SecretsManager: Add ability to list all encrypted values (#108512)

* list all encrypted values and count

* separate interfaces

* add time filter to global queries

* fix lint

pkg/registry/apis/secret/contracts/encryption.go

@@ -21,9 +21,20 @@ type EncryptedValue struct {
 	Updated       int64
 }
 
+// ListOpts defines pagination options for listing encrypted values.
+type ListOpts struct {
+	Limit  int64
+	Offset int64
+}
+
 type EncryptedValueStorage interface {
 	Create(ctx context.Context, namespace, name string, version int64, encryptedData []byte) (*EncryptedValue, error)
 	Update(ctx context.Context, namespace, name string, version int64, encryptedData []byte) error
 	Get(ctx context.Context, namespace, name string, version int64) (*EncryptedValue, error)
 	Delete(ctx context.Context, namespace, name string, version int64) error
 }
+
+type GlobalEncryptedValueStorage interface {
+	ListAll(ctx context.Context, opts ListOpts, untilTime *int64) ([]*EncryptedValue, error)
+	CountAll(ctx context.Context, untilTime *int64) (int64, error)
+}

pkg/registry/apis/secret/testutils/testutils.go

@@ -107,6 +107,10 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	encryptedValueStorage, err := encryptionstorage.ProvideEncryptedValueStorage(database, tracer)
 	require.NoError(t, err)
 
+	// Initialize global encrypted value storage with a fake db
+	globalEncryptedValueStorage, err := encryptionstorage.ProvideGlobalEncryptedValueStorage(database, tracer)
+	require.NoError(t, err)
+
 	sqlKeeper := sqlkeeper.NewSQLKeeper(tracer, encryptionManager, encryptedValueStorage, nil)
 
 	var keeperService contracts.KeeperService = newKeeperServiceWrapper(sqlKeeper)
@@ -125,26 +129,28 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut {
 	decryptService := decrypt.ProvideDecryptService(decryptStorage)
 
 	return Sut{
-		SecureValueService:         secureValueService,
+		SecureValueService:          secureValueService,
-		SecureValueMetadataStorage: secureValueMetadataStorage,
+		SecureValueMetadataStorage:  secureValueMetadataStorage,
-		DecryptStorage:             decryptStorage,
+		DecryptStorage:              decryptStorage,
-		DecryptService:             decryptService,
+		DecryptService:              decryptService,
-		EncryptedValueStorage:      encryptedValueStorage,
+		EncryptedValueStorage:       encryptedValueStorage,
-		SQLKeeper:                  sqlKeeper,
+		GlobalEncryptedValueStorage: globalEncryptedValueStorage,
-		Database:                   database,
+		SQLKeeper:                   sqlKeeper,
-		AccessClient:               accessClient,
+		Database:                    database,
+		AccessClient:                accessClient,
 	}
 }
 
 type Sut struct {
-	SecureValueService         contracts.SecureValueService
+	SecureValueService          contracts.SecureValueService
-	SecureValueMetadataStorage contracts.SecureValueMetadataStorage
+	SecureValueMetadataStorage  contracts.SecureValueMetadataStorage
-	DecryptStorage             contracts.DecryptStorage
+	DecryptStorage              contracts.DecryptStorage
-	DecryptService             contracts.DecryptService
+	DecryptService              contracts.DecryptService
-	EncryptedValueStorage      contracts.EncryptedValueStorage
+	EncryptedValueStorage       contracts.EncryptedValueStorage
-	SQLKeeper                  *sqlkeeper.SQLKeeper
+	GlobalEncryptedValueStorage contracts.GlobalEncryptedValueStorage
-	Database                   *database.Database
+	SQLKeeper                   *sqlkeeper.SQLKeeper
-	AccessClient               types.AccessClient
+	Database                    *database.Database
+	AccessClient                types.AccessClient
 }
 
 type CreateSvConfig struct {

pkg/storage/secret/encryption/data/encrypted_value_count_all.sql

@@ -0,0 +1,7 @@
+SELECT COUNT(*) AS count
+FROM
+  {{ .Ident "secret_encrypted_value" }}
+{{ if .HasUntilTime }}
+WHERE {{ .Ident "created" }} <= {{ .Arg .UntilTime }}
+{{ end }}
+;

pkg/storage/secret/encryption/data/encrypted_value_list_all.sql

@@ -0,0 +1,17 @@
+SELECT
+  {{ .Ident "namespace" }},
+  {{ .Ident "name" }},
+  {{ .Ident "version" }},
+  {{ .Ident "encrypted_data" }},
+  {{ .Ident "created" }},
+  {{ .Ident "updated" }}
+FROM
+  {{ .Ident "secret_encrypted_value" }}
+{{ if .HasUntilTime }}
+WHERE {{ .Ident "created" }} <= {{ .Arg .UntilTime }}
+{{ end }}
+ORDER BY {{ .Ident "created" }} ASC
+{{ if (gt .Limit 0) }}
+LIMIT {{ .Arg .Limit }} OFFSET {{ .Arg .Offset }}
+{{ end }}
+;

pkg/storage/secret/encryption/encrypted_value_store.go

@@ -206,3 +206,126 @@ func (s *encryptedValStorage) Delete(ctx context.Context, namespace, name string
 
 	return nil
 }
+
+type globalEncryptedValStorage struct {
+	db      contracts.Database
+	dialect sqltemplate.Dialect
+	tracer  trace.Tracer
+}
+
+func ProvideGlobalEncryptedValueStorage(
+	db contracts.Database,
+	tracer trace.Tracer,
+) (contracts.GlobalEncryptedValueStorage, error) {
+	return &globalEncryptedValStorage{
+		db:      db,
+		dialect: sqltemplate.DialectForDriver(db.DriverName()),
+		tracer:  tracer,
+	}, nil
+}
+
+func (s *globalEncryptedValStorage) ListAll(ctx context.Context, opts contracts.ListOpts, untilTime *int64) ([]*contracts.EncryptedValue, error) {
+	attrs := []attribute.KeyValue{
+		attribute.Int64("limit", opts.Limit),
+		attribute.Int64("offset", opts.Offset),
+	}
+	if untilTime != nil {
+		attrs = append(attrs, attribute.Int64("untilTime", *untilTime))
+	}
+	ctx, span := s.tracer.Start(ctx, "GlobalEncryptedValueStorage.CountAll", trace.WithAttributes(attrs...))
+	defer span.End()
+
+	req := listAllEncryptedValues{
+		SQLTemplate: sqltemplate.New(s.dialect),
+		Limit:       opts.Limit,
+		Offset:      opts.Offset,
+	}
+	if untilTime != nil {
+		req.HasUntilTime = true
+		req.UntilTime = *untilTime
+	}
+
+	query, err := sqltemplate.Execute(sqlEncryptedValueListAll, req)
+	if err != nil {
+		return nil, fmt.Errorf("execute template %q: %w", sqlEncryptedValueListAll.Name(), err)
+	}
+
+	rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...)
+	if err != nil {
+		return nil, fmt.Errorf("listing encrypted values %q: %w", sqlEncryptedValueListAll.Name(), err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	encryptedValues := make([]*contracts.EncryptedValue, 0)
+	for rows.Next() {
+		var row EncryptedValue
+		err = rows.Scan(
+			&row.Namespace,
+			&row.Name,
+			&row.Version,
+			&row.EncryptedData,
+			&row.Created,
+			&row.Updated,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("error reading data key row: %w", err)
+		}
+
+		encryptedValues = append(encryptedValues, &contracts.EncryptedValue{
+			Namespace:     row.Namespace,
+			Name:          row.Name,
+			Version:       row.Version,
+			EncryptedData: row.EncryptedData,
+			Created:       row.Created,
+			Updated:       row.Updated,
+		})
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("read rows error: %w", err)
+	}
+
+	return encryptedValues, nil
+}
+
+func (s *globalEncryptedValStorage) CountAll(ctx context.Context, untilTime *int64) (int64, error) {
+	attrs := []attribute.KeyValue{}
+	if untilTime != nil {
+		attrs = append(attrs, attribute.Int64("untilTime", *untilTime))
+	}
+	ctx, span := s.tracer.Start(ctx, "GlobalEncryptedValueStorage.CountAll", trace.WithAttributes(attrs...))
+	defer span.End()
+
+	req := countAllEncryptedValues{
+		SQLTemplate: sqltemplate.New(s.dialect),
+	}
+	if untilTime != nil {
+		req.HasUntilTime = true
+		req.UntilTime = *untilTime
+	}
+
+	query, err := sqltemplate.Execute(sqlEncryptedValueCountAll, req)
+	if err != nil {
+		return 0, fmt.Errorf("execute template %q: %w", sqlEncryptedValueCountAll.Name(), err)
+	}
+
+	rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...)
+	if err != nil {
+		return 0, fmt.Errorf("getting row: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	if !rows.Next() {
+		return 0, fmt.Errorf("no rows returned when counting encrypted values")
+	}
+
+	var count int64
+	err = rows.Scan(&count)
+	if err != nil {
+		return 0, fmt.Errorf("failed to scan encrypted value row: %w", err)
+	}
+	if err := rows.Err(); err != nil {
+		return 0, fmt.Errorf("read rows error: %w", err)
+	}
+
+	return count, nil
+}

pkg/storage/secret/encryption/encrypted_value_store_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"slices"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/testutils"
@@ -123,6 +124,77 @@ func TestEncryptedValueStoreImpl(t *testing.T) {
 		err := sut.EncryptedValueStorage.Delete(t.Context(), "test-namespace", "test-name", 1)
 		require.NoError(t, err)
 	})
+
+	t.Run("listing encrypted values returns them", func(t *testing.T) {
+		t.Parallel()
+
+		sut := testutils.Setup(t)
+		createdEvA, err := sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-a", "test-name", 1, []byte("test-data"))
+		require.NoError(t, err)
+
+		createdEvB, err := sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-b", "test-name", 1, []byte("test-data"))
+		require.NoError(t, err)
+
+		// List all encrypted values, without pagination
+		obtainedEVs, err := sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{}, nil)
+		require.NoError(t, err)
+		require.NotEmpty(t, obtainedEVs)
+		require.Len(t, obtainedEVs, 2)
+
+		obtainedEvA := obtainedEVs[0]
+		require.Equal(t, createdEvA.Namespace, obtainedEvA.Namespace)
+		require.Equal(t, createdEvA.Name, obtainedEvA.Name)
+		require.Equal(t, createdEvA.EncryptedData, obtainedEvA.EncryptedData)
+
+		// Test pagination by limiting the results to 1, offset by 0
+		obtainedEVs, err = sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{Limit: 1}, nil)
+		require.NoError(t, err)
+		require.NotEmpty(t, obtainedEVs)
+		require.Len(t, obtainedEVs, 1)
+
+		obtainedEvA = obtainedEVs[0]
+		require.Equal(t, createdEvA.Namespace, obtainedEvA.Namespace)
+		require.Equal(t, createdEvA.Name, obtainedEvA.Name)
+		require.Equal(t, createdEvA.EncryptedData, obtainedEvA.EncryptedData)
+
+		// Test pagination by limiting the results to 1, offset by 1
+		obtainedEVs, err = sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{Limit: 1, Offset: 1}, nil)
+		require.NoError(t, err)
+		require.NotEmpty(t, obtainedEVs)
+		require.Len(t, obtainedEVs, 1)
+
+		obtainedEvB := obtainedEVs[0]
+		require.Equal(t, createdEvB.Namespace, obtainedEvB.Namespace)
+		require.Equal(t, createdEvB.Name, obtainedEvB.Name)
+		require.Equal(t, createdEvB.EncryptedData, obtainedEvB.EncryptedData)
+
+		// List all encrypted values, until a certain time
+		pastTime := time.Now().Add(-1 * time.Hour).Unix()
+		obtainedEVs, err = sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{}, &pastTime)
+		require.NoError(t, err)
+		require.Empty(t, obtainedEVs)
+	})
+
+	t.Run("counting encrypted values returns their total", func(t *testing.T) {
+		t.Parallel()
+
+		sut := testutils.Setup(t)
+		_, err := sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-a", "test-name", 1, []byte("test-data"))
+		require.NoError(t, err)
+
+		_, err = sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-b", "test-name", 1, []byte("test-data"))
+		require.NoError(t, err)
+
+		count, err := sut.GlobalEncryptedValueStorage.CountAll(t.Context(), nil)
+		require.NoError(t, err)
+		require.Equal(t, int64(2), count)
+
+		// Count all encrypted values, until a certain time
+		pastTime := time.Now().Add(-1 * time.Hour).Unix()
+		count, err = sut.GlobalEncryptedValueStorage.CountAll(t.Context(), &pastTime)
+		require.NoError(t, err)
+		require.Equal(t, int64(0), count)
+	})
 }
 
 func TestStateMachine(t *testing.T) {

pkg/storage/secret/encryption/query.go

@@ -17,10 +17,12 @@ var (
 	sqlTemplates = template.Must(template.New("sql").ParseFS(sqlTemplatesFS, `data/*.sql`))
 
 	// The SQL Commands
-	sqlEncryptedValueCreate = mustTemplate("encrypted_value_create.sql")
+	sqlEncryptedValueCreate   = mustTemplate("encrypted_value_create.sql")
-	sqlEncryptedValueRead   = mustTemplate("encrypted_value_read.sql")
+	sqlEncryptedValueRead     = mustTemplate("encrypted_value_read.sql")
-	sqlEncryptedValueUpdate = mustTemplate("encrypted_value_update.sql")
+	sqlEncryptedValueUpdate   = mustTemplate("encrypted_value_update.sql")
-	sqlEncryptedValueDelete = mustTemplate("encrypted_value_delete.sql")
+	sqlEncryptedValueDelete   = mustTemplate("encrypted_value_delete.sql")
+	sqlEncryptedValueListAll  = mustTemplate("encrypted_value_list_all.sql")
+	sqlEncryptedValueCountAll = mustTemplate("encrypted_value_count_all.sql")
 
 	sqlDataKeyCreate      = mustTemplate("data_key_create.sql")
 	sqlDataKeyRead        = mustTemplate("data_key_read.sql")
@@ -93,6 +95,24 @@ func (r deleteEncryptedValue) Validate() error {
 	return nil // TODO
 }
 
+type listAllEncryptedValues struct {
+	sqltemplate.SQLTemplate
+	Limit        int64
+	Offset       int64
+	HasUntilTime bool
+	UntilTime    int64
+}
+
+func (r listAllEncryptedValues) Validate() error { return nil }
+
+type countAllEncryptedValues struct {
+	sqltemplate.SQLTemplate
+	HasUntilTime bool
+	UntilTime    int64
+}
+
+func (r countAllEncryptedValues) Validate() error { return nil }
+
 /*************************************/
 /**-- Data Key Queries --**/
 /*************************************/

pkg/storage/secret/encryption/query_test.go

@@ -10,6 +10,7 @@ import (
 )
 
 func TestEncryptedValueQueries(t *testing.T) {
+	untilTime := int64(1234)
 	mocks.CheckQuerySnapshots(t, mocks.TemplateTestSetup{
 		RootDir: "testdata",
 		Templates: map[*template.Template][]mocks.TemplateTestCase{
@@ -64,6 +65,63 @@ func TestEncryptedValueQueries(t *testing.T) {
 					},
 				},
 			},
+			sqlEncryptedValueListAll: {
+				{
+					Name: "list_limit_10_offset_0",
+					Data: &listAllEncryptedValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						Limit:        10,
+						Offset:       0,
+						HasUntilTime: false,
+					},
+				},
+				{
+					Name: "list_limit_10_offset_2",
+					Data: &listAllEncryptedValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						Limit:        10,
+						Offset:       2,
+						HasUntilTime: false,
+					},
+				},
+				{
+					Name: "list_all",
+					Data: &listAllEncryptedValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						Limit:        0,
+						Offset:       0,
+						HasUntilTime: false,
+					},
+				},
+				{
+					Name: "list_all_until_time",
+					Data: &listAllEncryptedValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						Limit:        0,
+						Offset:       0,
+						HasUntilTime: true,
+						UntilTime:    untilTime,
+					},
+				},
+			},
+			sqlEncryptedValueCountAll: {
+				{
+					Name: "count_all",
+					Data: &countAllEncryptedValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						HasUntilTime: false,
+						UntilTime:    0,
+					},
+				},
+				{
+					Name: "count_all_until_time",
+					Data: &countAllEncryptedValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						HasUntilTime: true,
+						UntilTime:    untilTime,
+					},
+				},
+			},
 		},
 	})
 }

pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all.sql

@@ -0,0 +1,4 @@
+SELECT COUNT(*) AS count
+FROM
+  `secret_encrypted_value`
+;

pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all_until_time.sql

@@ -0,0 +1,5 @@
+SELECT COUNT(*) AS count
+FROM
+  `secret_encrypted_value`
+WHERE `created` <= 1234
+;

pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all.sql

@@ -0,0 +1,11 @@
+SELECT
+  `namespace`,
+  `name`,
+  `version`,
+  `encrypted_data`,
+  `created`,
+  `updated`
+FROM
+  `secret_encrypted_value`
+ORDER BY `created` ASC
+;

pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all_until_time.sql

@@ -0,0 +1,12 @@
+SELECT
+  `namespace`,
+  `name`,
+  `version`,
+  `encrypted_data`,
+  `created`,
+  `updated`
+FROM
+  `secret_encrypted_value`
+WHERE `created` <= 1234
+ORDER BY `created` ASC
+;

pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_0.sql

@@ -0,0 +1,12 @@
+SELECT
+  `namespace`,
+  `name`,
+  `version`,
+  `encrypted_data`,
+  `created`,
+  `updated`
+FROM
+  `secret_encrypted_value`
+ORDER BY `created` ASC
+LIMIT 10 OFFSET 0
+;

pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_2.sql

@@ -0,0 +1,12 @@
+SELECT
+  `namespace`,
+  `name`,
+  `version`,
+  `encrypted_data`,
+  `created`,
+  `updated`
+FROM
+  `secret_encrypted_value`
+ORDER BY `created` ASC
+LIMIT 10 OFFSET 2
+;

pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all.sql

@@ -0,0 +1,4 @@
+SELECT COUNT(*) AS count
+FROM
+  "secret_encrypted_value"
+;

pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all_until_time.sql

@@ -0,0 +1,5 @@
+SELECT COUNT(*) AS count
+FROM
+  "secret_encrypted_value"
+WHERE "created" <= 1234
+;

pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all.sql

@@ -0,0 +1,11 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+ORDER BY "created" ASC
+;

pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all_until_time.sql

@@ -0,0 +1,12 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+WHERE "created" <= 1234
+ORDER BY "created" ASC
+;

pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_0.sql

@@ -0,0 +1,12 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+ORDER BY "created" ASC
+LIMIT 10 OFFSET 0
+;

pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_2.sql

@@ -0,0 +1,12 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+ORDER BY "created" ASC
+LIMIT 10 OFFSET 2
+;

pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all.sql

@@ -0,0 +1,4 @@
+SELECT COUNT(*) AS count
+FROM
+  "secret_encrypted_value"
+;

pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all_until_time.sql

@@ -0,0 +1,5 @@
+SELECT COUNT(*) AS count
+FROM
+  "secret_encrypted_value"
+WHERE "created" <= 1234
+;

pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all.sql

@@ -0,0 +1,11 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+ORDER BY "created" ASC
+;

pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all_until_time.sql

@@ -0,0 +1,12 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+WHERE "created" <= 1234
+ORDER BY "created" ASC
+;

pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_0.sql

@@ -0,0 +1,12 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+ORDER BY "created" ASC
+LIMIT 10 OFFSET 0
+;

pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_2.sql

@@ -0,0 +1,12 @@
+SELECT
+  "namespace",
+  "name",
+  "version",
+  "encrypted_data",
+  "created",
+  "updated"
+FROM
+  "secret_encrypted_value"
+ORDER BY "created" ASC
+LIMIT 10 OFFSET 2
+;