SecretsManager: Introduce keeper store (#105557)

* SecretsManager: Introduce secret database wrapper

Co-authored-by: PoorlyDefinedBehaviour <brunotj2015@hotmail.com>
Co-authored-by: Leandro Deveikis <leandro.deveikis@gmail.com>
Co-authored-by: Dana Axinte <53751979+dana-axinte@users.noreply.github.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* SecretsManager: Introduce db migrator with keeper table

Co-authored-by: PoorlyDefinedBehaviour <brunotj2015@hotmail.com>
Co-authored-by: Leandro Deveikis <leandro.deveikis@gmail.com>
Co-authored-by: Dana Axinte <53751979+dana-axinte@users.noreply.github.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* SecretsManager: Introduce keeper store

Co-authored-by: Leandro Deveikis <leandro.deveikis@gmail.com>
Co-authored-by: Dana Axinte <53751979+dana-axinte@users.noreply.github.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* new line

* without query listByNameSecureValue

* remove unused extractSecureValues for now

* SecretsManager: Add keeper integration tests

Co-authored-by: Leandro Deveikis <leandro.deveikis@gmail.com>
Co-authored-by: Dana Axinte <53751979+dana-axinte@users.noreply.github.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

---------

Co-authored-by: PoorlyDefinedBehaviour <brunotj2015@hotmail.com>
Co-authored-by: Leandro Deveikis <leandro.deveikis@gmail.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>
This commit is contained in:
Dana Axinte
2025-05-22 14:26:47 +01:00
committed by GitHub
co-authored by PoorlyDefinedBehaviour Leandro Deveikis Matheus Macabu
parent c5de567c8c
commit 7f2923d4ed
37 changed files with 2228 additions and 15 deletions
@@ -0,0 +1,27 @@
INSERT INTO {{ .Ident "secret_keeper" }} (
{{ .Ident "guid" }},
{{ .Ident "name" }},
{{ .Ident "namespace" }},
{{ .Ident "annotations" }},
{{ .Ident "labels" }},
{{ .Ident "created" }},
{{ .Ident "created_by" }},
{{ .Ident "updated" }},
{{ .Ident "updated_by" }},
{{ .Ident "description" }},
{{ .Ident "type" }},
{{ .Ident "payload" }}
) VALUES (
{{ .Arg .Row.GUID }},
{{ .Arg .Row.Name }},
{{ .Arg .Row.Namespace }},
{{ .Arg .Row.Annotations }},
{{ .Arg .Row.Labels }},
{{ .Arg .Row.Created }},
{{ .Arg .Row.CreatedBy }},
{{ .Arg .Row.Updated }},
{{ .Arg .Row.UpdatedBy }},
{{ .Arg .Row.Description }},
{{ .Arg .Row.Type }},
{{ .Arg .Row.Payload }}
);
@@ -0,0 +1,4 @@
DELETE FROM {{ .Ident "secret_keeper" }}
WHERE {{ .Ident "namespace" }} = {{ .Arg .Namespace }} AND
{{ .Ident "name" }} = {{ .Arg .Name }}
;
@@ -0,0 +1,18 @@
SELECT
{{ .Ident "guid" }},
{{ .Ident "name" }},
{{ .Ident "namespace" }},
{{ .Ident "annotations" }},
{{ .Ident "labels" }},
{{ .Ident "created" }},
{{ .Ident "created_by" }},
{{ .Ident "updated" }},
{{ .Ident "updated_by" }},
{{ .Ident "description" }},
{{ .Ident "type" }},
{{ .Ident "payload" }}
FROM
{{ .Ident "secret_keeper" }}
WHERE {{ .Ident "namespace" }} = {{ .Arg .Namespace }}
ORDER BY {{ .Ident "updated" }} DESC
;
@@ -0,0 +1,10 @@
{{/* this query is used to validate the keeper update or creation */}}
SELECT
{{ .Ident "name" }}
FROM
{{ .Ident "secret_keeper" }}
WHERE {{ .Ident "namespace" }} = {{ .Arg .Namespace }} AND
{{ .Ident "name" }} IN ({{ .ArgList .KeeperNames }})
{{ .SelectFor "UPDATE" }}
;
@@ -0,0 +1,21 @@
SELECT
{{ .Ident "guid" }},
{{ .Ident "name" }},
{{ .Ident "namespace" }},
{{ .Ident "annotations" }},
{{ .Ident "labels" }},
{{ .Ident "created" }},
{{ .Ident "created_by" }},
{{ .Ident "updated" }},
{{ .Ident "updated_by" }},
{{ .Ident "description" }},
{{ .Ident "type" }},
{{ .Ident "payload" }}
FROM
{{ .Ident "secret_keeper" }}
WHERE {{ .Ident "namespace" }} = {{ .Arg .Namespace }} AND
{{ .Ident "name" }} = {{ .Arg .Name }}
{{ if .IsForUpdate }}
{{ .SelectFor "UPDATE" }}
{{ end }}
;
@@ -0,0 +1,18 @@
UPDATE
{{ .Ident "secret_keeper" }}
SET
{{ .Ident "guid" }} = {{ .Arg .Row.GUID }},
{{ .Ident "name" }} = {{ .Arg .Row.Name }},
{{ .Ident "namespace" }} = {{ .Arg .Row.Namespace }},
{{ .Ident "annotations" }} = {{ .Arg .Row.Annotations }},
{{ .Ident "labels" }} = {{ .Arg .Row.Labels }},
{{ .Ident "created" }} = {{ .Arg .Row.Created }},
{{ .Ident "created_by" }} = {{ .Arg .Row.CreatedBy }},
{{ .Ident "updated" }} = {{ .Arg .Row.Updated }},
{{ .Ident "updated_by" }} = {{ .Arg .Row.UpdatedBy }},
{{ .Ident "description" }} = {{ .Arg .Row.Description }},
{{ .Ident "type" }} = {{ .Arg .Row.Type }},
{{ .Ident "payload" }} = {{ .Arg .Row.Payload }}
WHERE {{ .Ident "namespace" }} = {{ .Arg .Row.Namespace }} AND
{{ .Ident "name" }} = {{ .Arg .Row.Name }}
;
+248
View File
@@ -0,0 +1,248 @@
package metadata
import (
"encoding/json"
"fmt"
"time"
"github.com/google/uuid"
"github.com/grafana/grafana/pkg/apimachinery/utils"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
type keeperDB struct {
// Kubernetes Metadata
GUID string
Name string
Namespace string
Annotations string // map[string]string
Labels string // map[string]string
Created int64
CreatedBy string
Updated int64
UpdatedBy string
// Spec
Description string
Type string
Payload string
}
func (*keeperDB) TableName() string {
return migrator.TableNameKeeper
}
// toKubernetes maps a DB row into a Kubernetes resource (metadata + spec).
func (kp *keeperDB) toKubernetes() (*secretv0alpha1.Keeper, error) {
annotations := make(map[string]string, 0)
if kp.Annotations != "" {
if err := json.Unmarshal([]byte(kp.Annotations), &annotations); err != nil {
return nil, fmt.Errorf("failed to unmarshal annotations: %w", err)
}
}
labels := make(map[string]string, 0)
if kp.Labels != "" {
if err := json.Unmarshal([]byte(kp.Labels), &labels); err != nil {
return nil, fmt.Errorf("failed to unmarshal labels: %w", err)
}
}
resource := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: kp.Description,
},
}
// Obtain provider configs
provider := toProvider(secretv0alpha1.KeeperType(kp.Type), kp.Payload)
switch v := provider.(type) {
case *secretv0alpha1.AWSKeeperConfig:
resource.Spec.AWS = v
case *secretv0alpha1.AzureKeeperConfig:
resource.Spec.Azure = v
case *secretv0alpha1.GCPKeeperConfig:
resource.Spec.GCP = v
case *secretv0alpha1.HashiCorpKeeperConfig:
resource.Spec.HashiCorp = v
}
// Set all meta fields here for consistency.
meta, err := utils.MetaAccessor(resource)
if err != nil {
return nil, fmt.Errorf("failed to get meta accessor: %w", err)
}
updated := time.Unix(kp.Updated, 0).UTC()
meta.SetUID(types.UID(kp.GUID))
meta.SetName(kp.Name)
meta.SetNamespace(kp.Namespace)
meta.SetAnnotations(annotations)
meta.SetLabels(labels)
meta.SetCreatedBy(kp.CreatedBy)
meta.SetCreationTimestamp(metav1.NewTime(time.Unix(kp.Created, 0).UTC()))
meta.SetUpdatedBy(kp.UpdatedBy)
meta.SetUpdatedTimestamp(&updated)
meta.SetResourceVersionInt64(kp.Updated)
return resource, nil
}
// toKeeperCreateRow maps a Kubernetes resource into a DB row for new resources being created/inserted.
func toKeeperCreateRow(kp *secretv0alpha1.Keeper, actorUID string) (*keeperDB, error) {
row, err := toKeeperRow(kp)
if err != nil {
return nil, fmt.Errorf("failed to map to row: %w", err)
}
now := time.Now().UTC().Unix()
row.GUID = uuid.New().String()
row.Created = now
row.CreatedBy = actorUID
row.Updated = now
row.UpdatedBy = actorUID
return row, nil
}
// toKeeperUpdateRow maps a Kubernetes resource into a DB row for existing resources being updated.
func toKeeperUpdateRow(currentRow *keeperDB, newKeeper *secretv0alpha1.Keeper, actorUID string) (*keeperDB, error) {
row, err := toKeeperRow(newKeeper)
if err != nil {
return nil, fmt.Errorf("failed to map to row: %w", err)
}
now := time.Now().UTC().Unix()
row.GUID = currentRow.GUID
row.Created = currentRow.Created
row.CreatedBy = currentRow.CreatedBy
row.Updated = now
row.UpdatedBy = actorUID
return row, nil
}
// toKeeperRow maps a Kubernetes Keeper resource into a Keeper DB row.
func toKeeperRow(kp *secretv0alpha1.Keeper) (*keeperDB, error) {
var annotations string
if len(kp.Annotations) > 0 {
cleanedAnnotations := xkube.CleanAnnotations(kp.Annotations)
if len(cleanedAnnotations) > 0 {
kp.Annotations = make(map[string]string) // Safety: reset to prohibit use of kp.Annotations further.
encodedAnnotations, err := json.Marshal(cleanedAnnotations)
if err != nil {
return nil, fmt.Errorf("failed to encode annotations: %w", err)
}
annotations = string(encodedAnnotations)
}
}
var labels string
if len(kp.Labels) > 0 {
encodedLabels, err := json.Marshal(kp.Labels)
if err != nil {
return nil, fmt.Errorf("failed to encode labels: %w", err)
}
labels = string(encodedLabels)
}
meta, err := utils.MetaAccessor(kp)
if err != nil {
return nil, fmt.Errorf("failed to get meta accessor: %w", err)
}
if meta.GetFolder() != "" {
return nil, fmt.Errorf("folders are not supported")
}
updatedTimestamp, err := meta.GetResourceVersionInt64()
if err != nil {
return nil, fmt.Errorf("failed to get resource version: %w", err)
}
keeperType, keeperPayload, err := toTypeAndPayload(kp)
if err != nil {
return nil, fmt.Errorf("failed to obtain type and payload: %w", err)
}
return &keeperDB{
// Kubernetes Metadata
GUID: string(kp.UID),
Name: kp.Name,
Namespace: kp.Namespace,
Annotations: annotations,
Labels: labels,
Created: meta.GetCreationTimestamp().Unix(),
CreatedBy: meta.GetCreatedBy(),
Updated: updatedTimestamp,
UpdatedBy: meta.GetUpdatedBy(),
// Spec
Description: kp.Spec.Description,
Type: keeperType.String(),
Payload: keeperPayload,
}, nil
}
// toTypeAndPayload obtain keeper type and payload from a Kubernetes Keeper resource.
// TODO: Move as method of KeeperSpec
func toTypeAndPayload(kp *secretv0alpha1.Keeper) (secretv0alpha1.KeeperType, string, error) {
if kp.Spec.AWS != nil {
payload, err := json.Marshal(kp.Spec.AWS.AWSCredentials)
return secretv0alpha1.AWSKeeperType, string(payload), err
} else if kp.Spec.Azure != nil {
payload, err := json.Marshal(kp.Spec.Azure)
return secretv0alpha1.AzureKeeperType, string(payload), err
} else if kp.Spec.GCP != nil {
payload, err := json.Marshal(kp.Spec.GCP)
return secretv0alpha1.GCPKeeperType, string(payload), err
} else if kp.Spec.HashiCorp != nil {
payload, err := json.Marshal(kp.Spec.HashiCorp)
return secretv0alpha1.HashiCorpKeeperType, string(payload), err
}
return "", "", fmt.Errorf("no keeper type found")
}
// toProvider maps a KeeperType and payload into a provider config struct.
// TODO: Move as method of KeeperType
func toProvider(keeperType secretv0alpha1.KeeperType, payload string) secretv0alpha1.KeeperConfig {
switch keeperType {
case secretv0alpha1.AWSKeeperType:
aws := &secretv0alpha1.AWSKeeperConfig{}
if err := json.Unmarshal([]byte(payload), aws); err != nil {
return nil
}
return aws
case secretv0alpha1.AzureKeeperType:
azure := &secretv0alpha1.AzureKeeperConfig{}
if err := json.Unmarshal([]byte(payload), azure); err != nil {
return nil
}
return azure
case secretv0alpha1.GCPKeeperType:
gcp := &secretv0alpha1.GCPKeeperConfig{}
if err := json.Unmarshal([]byte(payload), gcp); err != nil {
return nil
}
return gcp
case secretv0alpha1.HashiCorpKeeperType:
hashicorp := &secretv0alpha1.HashiCorpKeeperConfig{}
if err := json.Unmarshal([]byte(payload), hashicorp); err != nil {
return nil
}
return hashicorp
default:
return nil
}
}
+241 -15
View File
@@ -2,50 +2,276 @@ package metadata
import (
"context"
"fmt"
claims "github.com/grafana/authlib/types"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
)
func ProvideKeeperMetadataStorage(db db.DB, features featuremgmt.FeatureToggles, accessClient claims.AccessClient) (contracts.KeeperMetadataStorage, error) {
// keeperMetadataStorage is the actual implementation of the keeper metadata storage.
type keeperMetadataStorage struct {
db contracts.Database
dialect sqltemplate.Dialect
}
var _ contracts.KeeperMetadataStorage = (*keeperMetadataStorage)(nil)
func ProvideKeeperMetadataStorage(db contracts.Database, features featuremgmt.FeatureToggles) (contracts.KeeperMetadataStorage, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) ||
!features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
return &keeperMetadataStorage{}, nil
}
return &keeperMetadataStorage{db: db, accessClient: accessClient}, nil
}
// keeperMetadataStorage is the actual implementation of the keeper metadata storage.
type keeperMetadataStorage struct {
db db.DB
accessClient claims.AccessClient
return &keeperMetadataStorage{
db: db,
dialect: sqltemplate.DialectForDriver(db.DriverName()),
}, nil
}
func (s *keeperMetadataStorage) Create(ctx context.Context, keeper *secretv0alpha1.Keeper, actorUID string) (*secretv0alpha1.Keeper, error) {
return nil, nil
row, err := toKeeperCreateRow(keeper, actorUID)
if err != nil {
return nil, fmt.Errorf("failed to create row: %w", err)
}
req := createKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Row: row,
}
query, err := sqltemplate.Execute(sqlKeeperCreate, req)
if err != nil {
return nil, fmt.Errorf("execute template %q: %w", sqlKeeperCreate.Name(), err)
}
err = s.db.Transaction(ctx, func(ctx context.Context) error {
result, err := s.db.ExecContext(ctx, query, req.GetArgs()...)
if err != nil {
return fmt.Errorf("inserting row: %w", err)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return fmt.Errorf("getting rows affected: %w", err)
}
if rowsAffected != 1 {
return fmt.Errorf("expected 1 row affected, got %d for %s on %s", rowsAffected, keeper.Name, keeper.Namespace)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("db failure: %w", err)
}
createdKeeper, err := row.toKubernetes()
if err != nil {
return nil, fmt.Errorf("failed to convert to kubernetes object: %w", err)
}
return createdKeeper, nil
}
func (s *keeperMetadataStorage) Read(ctx context.Context, namespace xkube.Namespace, name string, opts contracts.ReadOpts) (*secretv0alpha1.Keeper, error) {
return nil, nil
keeperDB, err := s.read(ctx, namespace.String(), name, opts)
if err != nil {
return nil, err
}
keeper, err := keeperDB.toKubernetes()
if err != nil {
return nil, fmt.Errorf("failed to convert to kubernetes object: %w", err)
}
return keeper, nil
}
func (s *keeperMetadataStorage) read(ctx context.Context, namespace, name string, opts contracts.ReadOpts) (*keeperDB, error) {
req := &readKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace,
Name: name,
IsForUpdate: opts.ForUpdate,
}
query, err := sqltemplate.Execute(sqlKeeperRead, req)
if err != nil {
return nil, fmt.Errorf("execute template %q: %w", sqlKeeperRead.Name(), err)
}
res, err := s.db.QueryContext(ctx, query, req.GetArgs()...)
if err != nil {
return nil, fmt.Errorf("getting row for %s in namespace %s: %w", name, namespace, err)
}
defer func() { _ = res.Close() }()
if !res.Next() {
return nil, contracts.ErrKeeperNotFound
}
var keeper keeperDB
err = res.Scan(
&keeper.GUID, &keeper.Name, &keeper.Namespace, &keeper.Annotations, &keeper.Labels, &keeper.Created,
&keeper.CreatedBy, &keeper.Updated, &keeper.UpdatedBy, &keeper.Description, &keeper.Type, &keeper.Payload,
)
if err != nil {
return nil, fmt.Errorf("failed to scan keeper row: %w", err)
}
if err := res.Err(); err != nil {
return nil, fmt.Errorf("read rows error: %w", err)
}
return &keeper, nil
}
func (s *keeperMetadataStorage) Update(ctx context.Context, newKeeper *secretv0alpha1.Keeper, actorUID string) (*secretv0alpha1.Keeper, error) {
return nil, nil
var newRow *keeperDB
err := s.db.Transaction(ctx, func(ctx context.Context) error {
// Read old value first.
oldKeeperRow, err := s.read(ctx, newKeeper.Namespace, newKeeper.Name, contracts.ReadOpts{ForUpdate: true})
if err != nil {
return err
}
// Generate an update row model.
var updateErr error
newRow, updateErr = toKeeperUpdateRow(oldKeeperRow, newKeeper, actorUID)
if updateErr != nil {
return fmt.Errorf("failed to map into update row: %w", updateErr)
}
// Update query with new model.
req := &updateKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Row: newRow,
}
query, err := sqltemplate.Execute(sqlKeeperUpdate, req)
if err != nil {
return fmt.Errorf("execute template %q: %w", sqlKeeperUpdate.Name(), err)
}
result, err := s.db.ExecContext(ctx, query, req.GetArgs()...)
if err != nil {
return fmt.Errorf("updating row: %w", err)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return fmt.Errorf("getting rows affected: %w", err)
}
if rowsAffected != 1 {
return fmt.Errorf("expected 1 row affected, got %d for %s on %s", rowsAffected, newKeeper.Name, newKeeper.Namespace)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("db failure: %w", err)
}
keeper, err := newRow.toKubernetes()
if err != nil {
return nil, fmt.Errorf("failed to convert to kubernetes object: %w", err)
}
return keeper, nil
}
func (s *keeperMetadataStorage) Delete(ctx context.Context, namespace xkube.Namespace, name string) error {
req := deleteKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
Name: name,
}
query, err := sqltemplate.Execute(sqlKeeperDelete, req)
if err != nil {
return fmt.Errorf("execute template %q: %w", sqlKeeperDelete.Name(), err)
}
result, err := s.db.ExecContext(ctx, query, req.GetArgs()...)
if err != nil {
return fmt.Errorf("deleting row: %w", err)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return fmt.Errorf("getting rows affected: %w", err)
}
if rowsAffected == 0 {
return contracts.ErrKeeperNotFound
} else if rowsAffected != 1 {
return fmt.Errorf("expected 1 row affected, got %d for %s on %s", rowsAffected, name, namespace)
}
return nil
}
func (s *keeperMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) ([]secretv0alpha1.Keeper, error) {
return nil, nil
req := listKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
}
query, err := sqltemplate.Execute(sqlKeeperList, req)
if err != nil {
return nil, fmt.Errorf("execute template %q: %w", sqlKeeperList.Name(), err)
}
rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...)
if err != nil {
return nil, fmt.Errorf("listing keepers %q: %w", sqlKeeperList.Name(), err)
}
defer func() { _ = rows.Close() }()
keepers := make([]secretv0alpha1.Keeper, 0)
for rows.Next() {
var row keeperDB
err = rows.Scan(
&row.GUID, &row.Name, &row.Namespace, &row.Annotations, &row.Labels, &row.Created,
&row.CreatedBy, &row.Updated, &row.UpdatedBy, &row.Description, &row.Type, &row.Payload,
)
if err != nil {
return nil, fmt.Errorf("error reading keeper row: %w", err)
}
keeper, err := row.toKubernetes()
if err != nil {
return nil, fmt.Errorf("failed to convert to kubernetes object: %w", err)
}
keepers = append(keepers, *keeper)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read rows error: %w", err)
}
return keepers, nil
}
func (s *keeperMetadataStorage) GetKeeperConfig(ctx context.Context, namespace string, name *string, opts contracts.ReadOpts) (secretv0alpha1.KeeperConfig, error) {
return nil, nil
// Check if keeper is the systemwide one.
if name == nil {
return nil, nil
}
// Load keeper config from metadata store, or TODO: keeper cache.
kp, err := s.read(ctx, namespace, *name, opts)
if err != nil {
return nil, err
}
keeperConfig := toProvider(secretv0alpha1.KeeperType(kp.Type), kp.Payload)
// TODO: this would be a good place to check if credentials are secure values and load them.
return keeperConfig, nil
}
@@ -0,0 +1,344 @@
package metadata
import (
"context"
"testing"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/storage/secret/database"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
"github.com/stretchr/testify/require"
)
func Test_KeeperMetadataStorage_GetKeeperConfig(t *testing.T) {
t.Parallel()
defaultKeeperName := "kp-test"
defaultKeeperNS := "default"
testKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "description",
AWS: &secretv0alpha1.AWSKeeperConfig{},
},
}
testKeeper.Name = defaultKeeperName
testKeeper.Namespace = defaultKeeperNS
t.Run("get the system keeper config", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
// get system keeper config
keeperConfig, err := keeperMetadataStorage.GetKeeperConfig(ctx, defaultKeeperNS, nil, contracts.ReadOpts{})
require.NoError(t, err)
require.Nil(t, keeperConfig)
})
t.Run("get test keeper config", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
//
_, err := keeperMetadataStorage.Create(ctx, testKeeper, "testuser")
require.NoError(t, err)
keeperConfig, err := keeperMetadataStorage.GetKeeperConfig(ctx, defaultKeeperNS, &defaultKeeperName, contracts.ReadOpts{})
require.NoError(t, err)
require.NotNil(t, keeperConfig)
require.NotEmpty(t, keeperConfig.Type())
})
t.Run("get test keeper config when listing", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
//
_, err := keeperMetadataStorage.Create(ctx, testKeeper, "testuser")
require.NoError(t, err)
keeperList, err := keeperMetadataStorage.List(ctx, xkube.Namespace(defaultKeeperNS))
require.NoError(t, err)
require.NotEmpty(t, keeperList)
require.Len(t, keeperList, 1)
keeper := keeperList[0]
require.Equal(t, "kp-test", keeper.Name)
require.Equal(t, "default", keeper.Namespace)
require.Equal(t, "description", keeper.Spec.Description)
})
t.Run("create a keeper and then delete it", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
keeperTest := "kp-test2"
keeperNamespaceTest := "ns"
testKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "another description",
AWS: &secretv0alpha1.AWSKeeperConfig{},
},
}
testKeeper.Name = keeperTest
testKeeper.Namespace = keeperNamespaceTest
// create the keeper
_, err := keeperMetadataStorage.Create(ctx, testKeeper, "testuser")
require.NoError(t, err)
// we are able to get it
keeperConfig, err := keeperMetadataStorage.GetKeeperConfig(ctx, keeperNamespaceTest, &keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.NotNil(t, keeperConfig)
require.NotEmpty(t, keeperConfig.Type())
// now we delete it
delErr := keeperMetadataStorage.Delete(ctx, xkube.Namespace(keeperNamespaceTest), keeperTest)
require.NoError(t, delErr)
// and we shouldn't be able to get it again
_, getErr := keeperMetadataStorage.GetKeeperConfig(ctx, keeperNamespaceTest, &keeperTest, contracts.ReadOpts{})
require.Errorf(t, getErr, "keeper not found")
})
t.Run("create, update and validate keeper", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
keeperTest := "kp-test3"
keeperNamespaceTest := "ns"
// Create initial keeper
initialKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "initial description",
AWS: &secretv0alpha1.AWSKeeperConfig{},
},
}
initialKeeper.Name = keeperTest
initialKeeper.Namespace = keeperNamespaceTest
// Create the keeper
_, err := keeperMetadataStorage.Create(ctx, initialKeeper, "testuser")
require.NoError(t, err)
// Validate that the description was set
keeper, err := keeperMetadataStorage.Read(ctx, xkube.Namespace(keeperNamespaceTest), keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.Equal(t, "initial description", keeper.Spec.Description)
// Update the keeper with new values
updatedKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "updated description",
AWS: &secretv0alpha1.AWSKeeperConfig{},
},
}
updatedKeeper.Name = keeperTest
updatedKeeper.Namespace = keeperNamespaceTest
// Perform the update
_, err = keeperMetadataStorage.Update(ctx, updatedKeeper, "testuser")
require.NoError(t, err)
// Validate updated values
updatedConfig, err := keeperMetadataStorage.GetKeeperConfig(ctx, keeperNamespaceTest, &keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.NotNil(t, updatedConfig)
require.NotEmpty(t, updatedConfig.Type())
// Validate that the description was updated
updatedKeeper, err = keeperMetadataStorage.Read(ctx, xkube.Namespace(keeperNamespaceTest), keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.Equal(t, "updated description", updatedKeeper.Spec.Description)
})
t.Run("update keeper with different AWS configuration", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
keeperTest := "kp-test4"
keeperNamespaceTest := "ns"
// Create initial keeper with first AWS config
initialKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "initial description",
AWS: &secretv0alpha1.AWSKeeperConfig{
AWSCredentials: secretv0alpha1.AWSCredentials{
AccessKeyID: secretv0alpha1.CredentialValue{
ValueFromEnv: "AWS_ACCESS_KEY_ID_1",
},
SecretAccessKey: secretv0alpha1.CredentialValue{
ValueFromEnv: "AWS_SECRET_ACCESS_KEY_1",
},
KMSKeyID: "kms-key-id-1",
},
},
},
}
initialKeeper.Name = keeperTest
initialKeeper.Namespace = keeperNamespaceTest
// Create the keeper
_, err := keeperMetadataStorage.Create(ctx, initialKeeper, "testuser")
require.NoError(t, err)
// Verify initial AWS config
keeper, err := keeperMetadataStorage.Read(ctx, xkube.Namespace(keeperNamespaceTest), keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.Equal(t, "AWS_ACCESS_KEY_ID_1", keeper.Spec.AWS.AccessKeyID.ValueFromEnv)
require.Equal(t, "AWS_SECRET_ACCESS_KEY_1", keeper.Spec.AWS.SecretAccessKey.ValueFromEnv)
require.Equal(t, "kms-key-id-1", keeper.Spec.AWS.KMSKeyID)
// Update with new AWS config
updatedKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "updated description",
AWS: &secretv0alpha1.AWSKeeperConfig{
AWSCredentials: secretv0alpha1.AWSCredentials{
AccessKeyID: secretv0alpha1.CredentialValue{
ValueFromEnv: "AWS_ACCESS_KEY_ID_2",
},
SecretAccessKey: secretv0alpha1.CredentialValue{
ValueFromEnv: "AWS_SECRET_ACCESS_KEY_2",
},
KMSKeyID: "kms-key-id-2",
},
},
},
}
updatedKeeper.Name = keeperTest
updatedKeeper.Namespace = keeperNamespaceTest
// Perform the update
_, err = keeperMetadataStorage.Update(ctx, updatedKeeper, "testuser")
require.NoError(t, err)
// Verify updated AWS config
updatedKeeper, err = keeperMetadataStorage.Read(ctx, xkube.Namespace(keeperNamespaceTest), keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.Equal(t, "AWS_ACCESS_KEY_ID_2", updatedKeeper.Spec.AWS.AccessKeyID.ValueFromEnv)
require.Equal(t, "AWS_SECRET_ACCESS_KEY_2", updatedKeeper.Spec.AWS.SecretAccessKey.ValueFromEnv)
require.Equal(t, "kms-key-id-2", updatedKeeper.Spec.AWS.KMSKeyID)
})
t.Run("list keepers in empty namespace", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
keeperList, err := keeperMetadataStorage.List(ctx, "")
require.NoError(t, err)
require.Empty(t, keeperList)
})
t.Run("read non-existent keeper", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
_, err := keeperMetadataStorage.Read(ctx, "ns", "non-existent", contracts.ReadOpts{})
require.Error(t, err)
require.Equal(t, contracts.ErrKeeperNotFound, err)
})
t.Run("update keeper with different namespace", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
keeperTest := "kp-test5"
keeperNamespaceTest := "ns1"
// Create initial keeper
initialKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "initial description",
AWS: &secretv0alpha1.AWSKeeperConfig{
AWSCredentials: secretv0alpha1.AWSCredentials{
AccessKeyID: secretv0alpha1.CredentialValue{
ValueFromEnv: "AWS_ACCESS_KEY_ID",
},
SecretAccessKey: secretv0alpha1.CredentialValue{
ValueFromEnv: "AWS_SECRET_ACCESS_KEY",
},
},
},
},
}
initialKeeper.Name = keeperTest
initialKeeper.Namespace = keeperNamespaceTest
// Create the keeper
_, err := keeperMetadataStorage.Create(ctx, initialKeeper, "testuser")
require.NoError(t, err)
// Try to update with different namespace
updatedKeeper := initialKeeper.DeepCopy()
updatedKeeper.Namespace = "ns2"
updatedKeeper.Spec.Description = "updated description"
_, err = keeperMetadataStorage.Update(ctx, updatedKeeper, "testuser")
// should this return contracts.ErrKeeperNotFound directly?
// require.Equal(t, contracts.ErrKeeperNotFound, err)
require.Error(t, err, "db failure: keeper not found")
// Verify original keeper is unchanged
keeper, err := keeperMetadataStorage.Read(ctx, xkube.Namespace(keeperNamespaceTest), keeperTest, contracts.ReadOpts{})
require.NoError(t, err)
require.Equal(t, "initial description", keeper.Spec.Description)
})
t.Run("update non-existent keeper", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
keeperMetadataStorage := initStorage(t)
nonExistentKeeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "some description",
AWS: &secretv0alpha1.AWSKeeperConfig{},
},
}
nonExistentKeeper.Name = "non-existent"
nonExistentKeeper.Namespace = "ns"
_, err := keeperMetadataStorage.Update(ctx, nonExistentKeeper, "testuser")
require.Error(t, err, "db failure: keeper not found")
})
}
func initStorage(t *testing.T) contracts.KeeperMetadataStorage {
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
db := database.ProvideDatabase(testDB)
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
// Initialize the keeper storage
keeperMetadataStorage, err := ProvideKeeperMetadataStorage(db, features)
require.NoError(t, err)
return keeperMetadataStorage
}
+106
View File
@@ -0,0 +1,106 @@
package metadata
import (
"embed"
"fmt"
"text/template"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
)
var (
//go:embed data/*.sql
sqlTemplatesFS embed.FS
sqlTemplates = template.Must(template.New("sql").ParseFS(sqlTemplatesFS, `data/*.sql`))
// The SQL Commands
sqlKeeperCreate = mustTemplate("keeper_create.sql")
sqlKeeperRead = mustTemplate("keeper_read.sql")
sqlKeeperUpdate = mustTemplate("keeper_update.sql")
sqlKeeperList = mustTemplate("keeper_list.sql")
sqlKeeperDelete = mustTemplate("keeper_delete.sql")
sqlKeeperListByName = mustTemplate("keeper_listByName.sql")
)
func mustTemplate(filename string) *template.Template {
if t := sqlTemplates.Lookup(filename); t != nil {
return t
}
panic(fmt.Sprintf("template file not found: %s", filename))
}
/************************/
/**-- Keeper Queries --**/
/************************/
// Create
type createKeeper struct {
sqltemplate.SQLTemplate
Row *keeperDB
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r createKeeper) Validate() error {
return nil // TODO
}
// Read
type readKeeper struct {
sqltemplate.SQLTemplate
Namespace string
Name string
IsForUpdate bool
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r readKeeper) Validate() error {
return nil // TODO
}
// Update
type updateKeeper struct {
sqltemplate.SQLTemplate
Row *keeperDB
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r updateKeeper) Validate() error {
return nil // TODO
}
// List
type listKeeper struct {
sqltemplate.SQLTemplate
Namespace string
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r listKeeper) Validate() error {
return nil // TODO
}
// Delete
type deleteKeeper struct {
sqltemplate.SQLTemplate
Namespace string
Name string
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r deleteKeeper) Validate() error {
return nil // TODO
}
// This is used at keeper store to validate create & update operations
type listByNameKeeper struct {
sqltemplate.SQLTemplate
Namespace string
KeeperNames []string
}
// Validate is only used if we use `dbutil` from `unifiedstorage`
func (r listByNameKeeper) Validate() error {
return nil // TODO
}
+108
View File
@@ -0,0 +1,108 @@
package metadata
import (
"testing"
"text/template"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
)
func TestKeeperQueries(t *testing.T) {
mocks.CheckQuerySnapshots(t, mocks.TemplateTestSetup{
RootDir: "testdata",
Templates: map[*template.Template][]mocks.TemplateTestCase{
sqlKeeperCreate: {
{
Name: "create",
Data: &createKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Row: &keeperDB{
GUID: "abc",
Name: "name",
Namespace: "ns",
Annotations: `{"x":"XXXX"}`,
Labels: `{"a":"AAA", "b", "BBBB"}`,
Created: 1234,
CreatedBy: "user:ryan",
Updated: 5678,
UpdatedBy: "user:cameron",
Description: "description",
Type: "sql",
Payload: "",
},
},
},
},
sqlKeeperDelete: {
{
Name: "delete",
Data: &deleteKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Name: "name",
Namespace: "ns",
},
},
},
sqlKeeperList: {
{
Name: "list",
Data: &listKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Namespace: "ns",
},
},
},
sqlKeeperRead: {
{
Name: "read",
Data: &readKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Name: "name",
Namespace: "ns",
},
},
{
Name: "read-for-update",
Data: &readKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Name: "name",
Namespace: "ns",
IsForUpdate: true,
},
},
},
sqlKeeperUpdate: {
{
Name: "update",
Data: &updateKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Row: &keeperDB{
GUID: "abc",
Name: "name",
Namespace: "ns",
Annotations: `{"x":"XXXX"}`,
Labels: `{"a":"AAA", "b", "BBBB"}`,
Created: 1234,
CreatedBy: "user:ryan",
Updated: 5678,
UpdatedBy: "user:cameron",
Description: "description",
Type: "sql",
Payload: "",
},
},
},
},
sqlKeeperListByName: {
{
Name: "list",
Data: listByNameKeeper{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Namespace: "ns",
KeeperNames: []string{"a", "b"},
},
},
},
},
})
}
+27
View File
@@ -0,0 +1,27 @@
INSERT INTO `secret_keeper` (
`guid`,
`name`,
`namespace`,
`annotations`,
`labels`,
`created`,
`created_by`,
`updated`,
`updated_by`,
`description`,
`type`,
`payload`
) VALUES (
'abc',
'name',
'ns',
'{"x":"XXXX"}',
'{"a":"AAA", "b", "BBBB"}',
1234,
'user:ryan',
5678,
'user:cameron',
'description',
'sql',
''
);
+4
View File
@@ -0,0 +1,4 @@
DELETE FROM `secret_keeper`
WHERE `namespace` = 'ns' AND
`name` = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
SELECT
`guid`,
`name`,
`namespace`,
`annotations`,
`labels`,
`created`,
`created_by`,
`updated`,
`updated_by`,
`description`,
`type`,
`payload`
FROM
`secret_keeper`
WHERE `namespace` = 'ns'
ORDER BY `updated` DESC
;
+8
View File
@@ -0,0 +1,8 @@
SELECT
`name`
FROM
`secret_keeper`
WHERE `namespace` = 'ns' AND
`name` IN ('a', 'b')
FOR UPDATE
;
@@ -0,0 +1,19 @@
SELECT
`guid`,
`name`,
`namespace`,
`annotations`,
`labels`,
`created`,
`created_by`,
`updated`,
`updated_by`,
`description`,
`type`,
`payload`
FROM
`secret_keeper`
WHERE `namespace` = 'ns' AND
`name` = 'name'
FOR UPDATE
;
+18
View File
@@ -0,0 +1,18 @@
SELECT
`guid`,
`name`,
`namespace`,
`annotations`,
`labels`,
`created`,
`created_by`,
`updated`,
`updated_by`,
`description`,
`type`,
`payload`
FROM
`secret_keeper`
WHERE `namespace` = 'ns' AND
`name` = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
UPDATE
`secret_keeper`
SET
`guid` = 'abc',
`name` = 'name',
`namespace` = 'ns',
`annotations` = '{"x":"XXXX"}',
`labels` = '{"a":"AAA", "b", "BBBB"}',
`created` = 1234,
`created_by` = 'user:ryan',
`updated` = 5678,
`updated_by` = 'user:cameron',
`description` = 'description',
`type` = 'sql',
`payload` = ''
WHERE `namespace` = 'ns' AND
`name` = 'name'
;
+27
View File
@@ -0,0 +1,27 @@
INSERT INTO "secret_keeper" (
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
) VALUES (
'abc',
'name',
'ns',
'{"x":"XXXX"}',
'{"a":"AAA", "b", "BBBB"}',
1234,
'user:ryan',
5678,
'user:cameron',
'description',
'sql',
''
);
@@ -0,0 +1,4 @@
DELETE FROM "secret_keeper"
WHERE "namespace" = 'ns' AND
"name" = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
FROM
"secret_keeper"
WHERE "namespace" = 'ns'
ORDER BY "updated" DESC
;
@@ -0,0 +1,8 @@
SELECT
"name"
FROM
"secret_keeper"
WHERE "namespace" = 'ns' AND
"name" IN ('a', 'b')
FOR UPDATE
;
@@ -0,0 +1,19 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
FROM
"secret_keeper"
WHERE "namespace" = 'ns' AND
"name" = 'name'
FOR UPDATE
;
+18
View File
@@ -0,0 +1,18 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
FROM
"secret_keeper"
WHERE "namespace" = 'ns' AND
"name" = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
UPDATE
"secret_keeper"
SET
"guid" = 'abc',
"name" = 'name',
"namespace" = 'ns',
"annotations" = '{"x":"XXXX"}',
"labels" = '{"a":"AAA", "b", "BBBB"}',
"created" = 1234,
"created_by" = 'user:ryan',
"updated" = 5678,
"updated_by" = 'user:cameron',
"description" = 'description',
"type" = 'sql',
"payload" = ''
WHERE "namespace" = 'ns' AND
"name" = 'name'
;
+27
View File
@@ -0,0 +1,27 @@
INSERT INTO "secret_keeper" (
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
) VALUES (
'abc',
'name',
'ns',
'{"x":"XXXX"}',
'{"a":"AAA", "b", "BBBB"}',
1234,
'user:ryan',
5678,
'user:cameron',
'description',
'sql',
''
);
+4
View File
@@ -0,0 +1,4 @@
DELETE FROM "secret_keeper"
WHERE "namespace" = 'ns' AND
"name" = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
FROM
"secret_keeper"
WHERE "namespace" = 'ns'
ORDER BY "updated" DESC
;
@@ -0,0 +1,7 @@
SELECT
"name"
FROM
"secret_keeper"
WHERE "namespace" = 'ns' AND
"name" IN ('a', 'b')
;
@@ -0,0 +1,18 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
FROM
"secret_keeper"
WHERE "namespace" = 'ns' AND
"name" = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
SELECT
"guid",
"name",
"namespace",
"annotations",
"labels",
"created",
"created_by",
"updated",
"updated_by",
"description",
"type",
"payload"
FROM
"secret_keeper"
WHERE "namespace" = 'ns' AND
"name" = 'name'
;
+18
View File
@@ -0,0 +1,18 @@
UPDATE
"secret_keeper"
SET
"guid" = 'abc',
"name" = 'name',
"namespace" = 'ns',
"annotations" = '{"x":"XXXX"}',
"labels" = '{"a":"AAA", "b", "BBBB"}',
"created" = 1234,
"created_by" = 'user:ryan',
"updated" = 5678,
"updated_by" = 'user:cameron',
"description" = 'description',
"type" = 'sql',
"payload" = ''
WHERE "namespace" = 'ns' AND
"name" = 'name'
;