SecretsManager: Unify KMS across OSS and Enterprise (#108085)

* everything is compiling * tests passing * remove used object * write a test for secret key upgrades * misc cleanup * clean up some wording * lint issues * fix a typo * import hashicorp dependency explicitly * simplify oss kmsprovider package structure * consolidate current provider and available providers * add a new manager configuration test * fix hashivault import * fix import issue * fix unit tests * Update go.mod Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com> --------- Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>
2025-07-15 14:39:20 -04:00
parent 469f94028f
commit ab51794bdb
20 changed files with 631 additions and 242 deletions
@@ -1,22 +1,37 @@
 package setting

 import (
-	"regexp"
+	"strings"
+)

-	"github.com/grafana/grafana/pkg/services/kmsproviders"
+const (
+	ProviderPrefix        = "secrets_manager.encryption."
+	MisconfiguredProvider = "misconfigured"
 )

 type SecretsManagerSettings struct {
-	SecretKey          string
-	EncryptionProvider string
-	AvailableProviders []string
+	CurrentEncryptionProvider string
+
+	// ConfiguredKMSProviders is a map of KMS providers found in the config file. The keys are in the format of <provider>.<keyName>, and the values are a map of the properties in that section
+	// In OSS, the provider type can only be "secret_key". In Enterprise, it can additionally be one of: "aws_kms", "azure_keyvault", "google_kms", "hashicorp_vault"
+	ConfiguredKMSProviders map[string]map[string]string
 }

 func (cfg *Cfg) readSecretsManagerSettings() {
 	secretsMgmt := cfg.Raw.Section("secrets_manager")
-	cfg.SecretsManagement.EncryptionProvider = secretsMgmt.Key("encryption_provider").MustString(kmsproviders.Default)
+	cfg.SecretsManagement.CurrentEncryptionProvider = secretsMgmt.Key("encryption_provider").MustString(MisconfiguredProvider)

-	// TODO: These are not used yet by the secrets manager because we need to distentagle the dependencies with OSS.
-	cfg.SecretsManagement.SecretKey = secretsMgmt.Key("secret_key").MustString("")
-	cfg.SecretsManagement.AvailableProviders = regexp.MustCompile(`\s*,\s*`).Split(secretsMgmt.Key("available_encryption_providers").MustString(""), -1) // parse comma separated list
+	// Extract available KMS providers from configuration sections
+	providers := make(map[string]map[string]string)
+	for _, section := range cfg.Raw.Sections() {
+		sectionName := section.Name()
+		if strings.HasPrefix(sectionName, ProviderPrefix) {
+			// Extract the provider name (everything after the prefix)
+			providerName := strings.TrimPrefix(sectionName, ProviderPrefix)
+			if providerName != "" {
+				providers[providerName] = section.KeysHash()
+			}
+		}
+	}
+	cfg.SecretsManagement.ConfiguredKMSProviders = providers
 }
@@ -0,0 +1,173 @@
+package setting
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadSecretsManagerSettings(t *testing.T) {
+	t.Run("should parse basic encryption provider", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+encryption_provider = aws_kms
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, "aws_kms", cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Empty(t, cfg.SecretsManagement.ConfiguredKMSProviders)
+	})
+
+	t.Run("should parse single KMS provider configuration", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+encryption_provider = aws_kms.v1
+
+[secrets_manager.encryption.aws_kms.v1]
+region = us-east-1
+key_id = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, "aws_kms.v1", cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Len(t, cfg.SecretsManagement.ConfiguredKMSProviders, 1)
+
+		awsProvider := cfg.SecretsManagement.ConfiguredKMSProviders["aws_kms.v1"]
+		assert.Equal(t, "us-east-1", awsProvider["region"])
+		assert.Equal(t, "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012", awsProvider["key_id"])
+	})
+
+	t.Run("should parse multiple KMS providers", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+encryption_provider = aws_kms.v1
+
+[secrets_manager.encryption.aws_kms.v1]
+region = us-east-1
+key_id = arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
+
+[secrets_manager.encryption.azure_kv.v1]
+vault_url = https://myvault.vault.azure.net/
+key_name = mykey
+tenant_id = 12345678-1234-1234-1234-123456789012
+
+[secrets_manager.encryption.secret_key.v1]
+key = my-secret-key
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, "aws_kms.v1", cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Len(t, cfg.SecretsManagement.ConfiguredKMSProviders, 3)
+
+		// Check AWS KMS provider
+		awsProvider := cfg.SecretsManagement.ConfiguredKMSProviders["aws_kms.v1"]
+		assert.Equal(t, "us-east-1", awsProvider["region"])
+		assert.Equal(t, "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012", awsProvider["key_id"])
+
+		// Check Azure Key Vault provider
+		azureProvider := cfg.SecretsManagement.ConfiguredKMSProviders["azure_kv.v1"]
+		assert.Equal(t, "https://myvault.vault.azure.net/", azureProvider["vault_url"])
+		assert.Equal(t, "mykey", azureProvider["key_name"])
+		assert.Equal(t, "12345678-1234-1234-1234-123456789012", azureProvider["tenant_id"])
+
+		// Check secret key provider
+		secretProvider := cfg.SecretsManagement.ConfiguredKMSProviders["secret_key.v1"]
+		assert.Equal(t, "my-secret-key", secretProvider["key"])
+	})
+
+	t.Run("should default to misconfigured provider when no encryption_provider is set", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+# no encryption_provider setting
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, MisconfiguredProvider, cfg.SecretsManagement.CurrentEncryptionProvider)
+	})
+
+	t.Run("should handle empty sections gracefully", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+encryption_provider = empty_provider
+
+[secrets_manager.encryption.empty_provider]
+# empty section
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, "empty_provider", cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Len(t, cfg.SecretsManagement.ConfiguredKMSProviders, 1)
+
+		emptyProvider := cfg.SecretsManagement.ConfiguredKMSProviders["empty_provider"]
+		assert.NotNil(t, emptyProvider)
+		assert.Empty(t, emptyProvider)
+	})
+
+	t.Run("should ignore sections that don't match provider prefix", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+encryption_provider = aws_kms.v1
+
+[secrets_manager.encryption.valid_provider]
+key = value
+
+[secrets_manager.other_section]
+setting = value
+
+[completely_different_section]
+some_setting = some_value
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, "aws_kms.v1", cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Len(t, cfg.SecretsManagement.ConfiguredKMSProviders, 1)
+
+		validProvider := cfg.SecretsManagement.ConfiguredKMSProviders["valid_provider"]
+		assert.Equal(t, "value", validProvider["key"])
+	})
+
+	t.Run("should handle provider names with special characters", func(t *testing.T) {
+		iniContent := `
+[secrets_manager]
+encryption_provider = aws_kms.v1
+
+[secrets_manager.encryption.aws_kms.v1]
+region = us-west-2
+key_id = test-key
+
+[secrets_manager.encryption.azure_kv.v1]
+vault_url = https://test.vault.azure.net/
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, "aws_kms.v1", cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Len(t, cfg.SecretsManagement.ConfiguredKMSProviders, 2)
+
+		awsProvider := cfg.SecretsManagement.ConfiguredKMSProviders["aws_kms.v1"]
+		assert.Equal(t, "us-west-2", awsProvider["region"])
+		assert.Equal(t, "test-key", awsProvider["key_id"])
+
+		azureProvider := cfg.SecretsManagement.ConfiguredKMSProviders["azure_kv.v1"]
+		assert.Equal(t, "https://test.vault.azure.net/", azureProvider["vault_url"])
+	})
+
+	t.Run("should handle configuration with no secrets_manager section", func(t *testing.T) {
+		iniContent := `
+[server]
+domain = example.com
+`
+		cfg, err := NewCfgFromBytes([]byte(iniContent))
+		require.NoError(t, err)
+
+		assert.Equal(t, MisconfiguredProvider, cfg.SecretsManagement.CurrentEncryptionProvider)
+		assert.Empty(t, cfg.SecretsManagement.ConfiguredKMSProviders)
+	})
+}