From 2ea77a7c05e18d7c19e3b6307326fc1272346f52 Mon Sep 17 00:00:00 2001 From: Dana Axinte <53751979+dana-axinte@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:50:24 +0100 Subject: [PATCH 1/4] SecretsManager: Add ability to list all encrypted values (#108512) * list all encrypted values and count * separate interfaces * add time filter to global queries * fix lint --- .../apis/secret/contracts/encryption.go | 11 ++ .../apis/secret/testutils/testutils.go | 38 +++--- .../data/encrypted_value_count_all.sql | 7 + .../data/encrypted_value_list_all.sql | 17 +++ .../encryption/encrypted_value_store.go | 123 ++++++++++++++++++ .../encryption/encrypted_value_store_test.go | 72 ++++++++++ pkg/storage/secret/encryption/query.go | 28 +++- pkg/storage/secret/encryption/query_test.go | 58 +++++++++ ...l--encrypted_value_count_all-count_all.sql | 4 + ...d_value_count_all-count_all_until_time.sql | 5 + ...sql--encrypted_value_list_all-list_all.sql | 11 ++ ...ted_value_list_all-list_all_until_time.sql | 12 ++ ..._value_list_all-list_limit_10_offset_0.sql | 12 ++ ..._value_list_all-list_limit_10_offset_2.sql | 12 ++ ...s--encrypted_value_count_all-count_all.sql | 4 + ...d_value_count_all-count_all_until_time.sql | 5 + ...res--encrypted_value_list_all-list_all.sql | 11 ++ ...ted_value_list_all-list_all_until_time.sql | 12 ++ ..._value_list_all-list_limit_10_offset_0.sql | 12 ++ ..._value_list_all-list_limit_10_offset_2.sql | 12 ++ ...e--encrypted_value_count_all-count_all.sql | 4 + ...d_value_count_all-count_all_until_time.sql | 5 + ...ite--encrypted_value_list_all-list_all.sql | 11 ++ ...ted_value_list_all-list_all_until_time.sql | 12 ++ ..._value_list_all-list_limit_10_offset_0.sql | 12 ++ ..._value_list_all-list_limit_10_offset_2.sql | 12 ++ 26 files changed, 502 insertions(+), 20 deletions(-) create mode 100644 pkg/storage/secret/encryption/data/encrypted_value_count_all.sql create mode 100644 pkg/storage/secret/encryption/data/encrypted_value_list_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all_until_time.sql create mode 100755 pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all_until_time.sql create mode 100755 pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_0.sql create mode 100755 pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_2.sql create mode 100755 pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all_until_time.sql create mode 100755 pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all_until_time.sql create mode 100755 pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_0.sql create mode 100755 pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_2.sql create mode 100755 pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all_until_time.sql create mode 100755 pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all.sql create mode 100755 pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all_until_time.sql create mode 100755 pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_0.sql create mode 100755 pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_2.sql diff --git a/pkg/registry/apis/secret/contracts/encryption.go b/pkg/registry/apis/secret/contracts/encryption.go index eb5292ee794..f0596de9034 100644 --- a/pkg/registry/apis/secret/contracts/encryption.go +++ b/pkg/registry/apis/secret/contracts/encryption.go @@ -21,9 +21,20 @@ type EncryptedValue struct { Updated int64 } +// ListOpts defines pagination options for listing encrypted values. +type ListOpts struct { + Limit int64 + Offset int64 +} + type EncryptedValueStorage interface { Create(ctx context.Context, namespace, name string, version int64, encryptedData []byte) (*EncryptedValue, error) Update(ctx context.Context, namespace, name string, version int64, encryptedData []byte) error Get(ctx context.Context, namespace, name string, version int64) (*EncryptedValue, error) Delete(ctx context.Context, namespace, name string, version int64) error } + +type GlobalEncryptedValueStorage interface { + ListAll(ctx context.Context, opts ListOpts, untilTime *int64) ([]*EncryptedValue, error) + CountAll(ctx context.Context, untilTime *int64) (int64, error) +} diff --git a/pkg/registry/apis/secret/testutils/testutils.go b/pkg/registry/apis/secret/testutils/testutils.go index 478b9c63e84..448283abd09 100644 --- a/pkg/registry/apis/secret/testutils/testutils.go +++ b/pkg/registry/apis/secret/testutils/testutils.go @@ -107,6 +107,10 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut { encryptedValueStorage, err := encryptionstorage.ProvideEncryptedValueStorage(database, tracer) require.NoError(t, err) + // Initialize global encrypted value storage with a fake db + globalEncryptedValueStorage, err := encryptionstorage.ProvideGlobalEncryptedValueStorage(database, tracer) + require.NoError(t, err) + sqlKeeper := sqlkeeper.NewSQLKeeper(tracer, encryptionManager, encryptedValueStorage, nil) var keeperService contracts.KeeperService = newKeeperServiceWrapper(sqlKeeper) @@ -125,26 +129,28 @@ func Setup(t *testing.T, opts ...func(*SetupConfig)) Sut { decryptService := decrypt.ProvideDecryptService(decryptStorage) return Sut{ - SecureValueService: secureValueService, - SecureValueMetadataStorage: secureValueMetadataStorage, - DecryptStorage: decryptStorage, - DecryptService: decryptService, - EncryptedValueStorage: encryptedValueStorage, - SQLKeeper: sqlKeeper, - Database: database, - AccessClient: accessClient, + SecureValueService: secureValueService, + SecureValueMetadataStorage: secureValueMetadataStorage, + DecryptStorage: decryptStorage, + DecryptService: decryptService, + EncryptedValueStorage: encryptedValueStorage, + GlobalEncryptedValueStorage: globalEncryptedValueStorage, + SQLKeeper: sqlKeeper, + Database: database, + AccessClient: accessClient, } } type Sut struct { - SecureValueService contracts.SecureValueService - SecureValueMetadataStorage contracts.SecureValueMetadataStorage - DecryptStorage contracts.DecryptStorage - DecryptService contracts.DecryptService - EncryptedValueStorage contracts.EncryptedValueStorage - SQLKeeper *sqlkeeper.SQLKeeper - Database *database.Database - AccessClient types.AccessClient + SecureValueService contracts.SecureValueService + SecureValueMetadataStorage contracts.SecureValueMetadataStorage + DecryptStorage contracts.DecryptStorage + DecryptService contracts.DecryptService + EncryptedValueStorage contracts.EncryptedValueStorage + GlobalEncryptedValueStorage contracts.GlobalEncryptedValueStorage + SQLKeeper *sqlkeeper.SQLKeeper + Database *database.Database + AccessClient types.AccessClient } type CreateSvConfig struct { diff --git a/pkg/storage/secret/encryption/data/encrypted_value_count_all.sql b/pkg/storage/secret/encryption/data/encrypted_value_count_all.sql new file mode 100644 index 00000000000..5bfc5d2424f --- /dev/null +++ b/pkg/storage/secret/encryption/data/encrypted_value_count_all.sql @@ -0,0 +1,7 @@ +SELECT COUNT(*) AS count +FROM + {{ .Ident "secret_encrypted_value" }} +{{ if .HasUntilTime }} +WHERE {{ .Ident "created" }} <= {{ .Arg .UntilTime }} +{{ end }} +; diff --git a/pkg/storage/secret/encryption/data/encrypted_value_list_all.sql b/pkg/storage/secret/encryption/data/encrypted_value_list_all.sql new file mode 100644 index 00000000000..d318517346d --- /dev/null +++ b/pkg/storage/secret/encryption/data/encrypted_value_list_all.sql @@ -0,0 +1,17 @@ +SELECT + {{ .Ident "namespace" }}, + {{ .Ident "name" }}, + {{ .Ident "version" }}, + {{ .Ident "encrypted_data" }}, + {{ .Ident "created" }}, + {{ .Ident "updated" }} +FROM + {{ .Ident "secret_encrypted_value" }} +{{ if .HasUntilTime }} +WHERE {{ .Ident "created" }} <= {{ .Arg .UntilTime }} +{{ end }} +ORDER BY {{ .Ident "created" }} ASC +{{ if (gt .Limit 0) }} +LIMIT {{ .Arg .Limit }} OFFSET {{ .Arg .Offset }} +{{ end }} +; diff --git a/pkg/storage/secret/encryption/encrypted_value_store.go b/pkg/storage/secret/encryption/encrypted_value_store.go index cd298591b31..3a10d01b957 100644 --- a/pkg/storage/secret/encryption/encrypted_value_store.go +++ b/pkg/storage/secret/encryption/encrypted_value_store.go @@ -206,3 +206,126 @@ func (s *encryptedValStorage) Delete(ctx context.Context, namespace, name string return nil } + +type globalEncryptedValStorage struct { + db contracts.Database + dialect sqltemplate.Dialect + tracer trace.Tracer +} + +func ProvideGlobalEncryptedValueStorage( + db contracts.Database, + tracer trace.Tracer, +) (contracts.GlobalEncryptedValueStorage, error) { + return &globalEncryptedValStorage{ + db: db, + dialect: sqltemplate.DialectForDriver(db.DriverName()), + tracer: tracer, + }, nil +} + +func (s *globalEncryptedValStorage) ListAll(ctx context.Context, opts contracts.ListOpts, untilTime *int64) ([]*contracts.EncryptedValue, error) { + attrs := []attribute.KeyValue{ + attribute.Int64("limit", opts.Limit), + attribute.Int64("offset", opts.Offset), + } + if untilTime != nil { + attrs = append(attrs, attribute.Int64("untilTime", *untilTime)) + } + ctx, span := s.tracer.Start(ctx, "GlobalEncryptedValueStorage.CountAll", trace.WithAttributes(attrs...)) + defer span.End() + + req := listAllEncryptedValues{ + SQLTemplate: sqltemplate.New(s.dialect), + Limit: opts.Limit, + Offset: opts.Offset, + } + if untilTime != nil { + req.HasUntilTime = true + req.UntilTime = *untilTime + } + + query, err := sqltemplate.Execute(sqlEncryptedValueListAll, req) + if err != nil { + return nil, fmt.Errorf("execute template %q: %w", sqlEncryptedValueListAll.Name(), err) + } + + rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...) + if err != nil { + return nil, fmt.Errorf("listing encrypted values %q: %w", sqlEncryptedValueListAll.Name(), err) + } + defer func() { _ = rows.Close() }() + + encryptedValues := make([]*contracts.EncryptedValue, 0) + for rows.Next() { + var row EncryptedValue + err = rows.Scan( + &row.Namespace, + &row.Name, + &row.Version, + &row.EncryptedData, + &row.Created, + &row.Updated, + ) + if err != nil { + return nil, fmt.Errorf("error reading data key row: %w", err) + } + + encryptedValues = append(encryptedValues, &contracts.EncryptedValue{ + Namespace: row.Namespace, + Name: row.Name, + Version: row.Version, + EncryptedData: row.EncryptedData, + Created: row.Created, + Updated: row.Updated, + }) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read rows error: %w", err) + } + + return encryptedValues, nil +} + +func (s *globalEncryptedValStorage) CountAll(ctx context.Context, untilTime *int64) (int64, error) { + attrs := []attribute.KeyValue{} + if untilTime != nil { + attrs = append(attrs, attribute.Int64("untilTime", *untilTime)) + } + ctx, span := s.tracer.Start(ctx, "GlobalEncryptedValueStorage.CountAll", trace.WithAttributes(attrs...)) + defer span.End() + + req := countAllEncryptedValues{ + SQLTemplate: sqltemplate.New(s.dialect), + } + if untilTime != nil { + req.HasUntilTime = true + req.UntilTime = *untilTime + } + + query, err := sqltemplate.Execute(sqlEncryptedValueCountAll, req) + if err != nil { + return 0, fmt.Errorf("execute template %q: %w", sqlEncryptedValueCountAll.Name(), err) + } + + rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...) + if err != nil { + return 0, fmt.Errorf("getting row: %w", err) + } + defer func() { _ = rows.Close() }() + + if !rows.Next() { + return 0, fmt.Errorf("no rows returned when counting encrypted values") + } + + var count int64 + err = rows.Scan(&count) + if err != nil { + return 0, fmt.Errorf("failed to scan encrypted value row: %w", err) + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("read rows error: %w", err) + } + + return count, nil +} diff --git a/pkg/storage/secret/encryption/encrypted_value_store_test.go b/pkg/storage/secret/encryption/encrypted_value_store_test.go index 18b67a651fb..e5f987da4dc 100644 --- a/pkg/storage/secret/encryption/encrypted_value_store_test.go +++ b/pkg/storage/secret/encryption/encrypted_value_store_test.go @@ -4,6 +4,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" "github.com/grafana/grafana/pkg/registry/apis/secret/testutils" @@ -123,6 +124,77 @@ func TestEncryptedValueStoreImpl(t *testing.T) { err := sut.EncryptedValueStorage.Delete(t.Context(), "test-namespace", "test-name", 1) require.NoError(t, err) }) + + t.Run("listing encrypted values returns them", func(t *testing.T) { + t.Parallel() + + sut := testutils.Setup(t) + createdEvA, err := sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-a", "test-name", 1, []byte("test-data")) + require.NoError(t, err) + + createdEvB, err := sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-b", "test-name", 1, []byte("test-data")) + require.NoError(t, err) + + // List all encrypted values, without pagination + obtainedEVs, err := sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{}, nil) + require.NoError(t, err) + require.NotEmpty(t, obtainedEVs) + require.Len(t, obtainedEVs, 2) + + obtainedEvA := obtainedEVs[0] + require.Equal(t, createdEvA.Namespace, obtainedEvA.Namespace) + require.Equal(t, createdEvA.Name, obtainedEvA.Name) + require.Equal(t, createdEvA.EncryptedData, obtainedEvA.EncryptedData) + + // Test pagination by limiting the results to 1, offset by 0 + obtainedEVs, err = sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{Limit: 1}, nil) + require.NoError(t, err) + require.NotEmpty(t, obtainedEVs) + require.Len(t, obtainedEVs, 1) + + obtainedEvA = obtainedEVs[0] + require.Equal(t, createdEvA.Namespace, obtainedEvA.Namespace) + require.Equal(t, createdEvA.Name, obtainedEvA.Name) + require.Equal(t, createdEvA.EncryptedData, obtainedEvA.EncryptedData) + + // Test pagination by limiting the results to 1, offset by 1 + obtainedEVs, err = sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{Limit: 1, Offset: 1}, nil) + require.NoError(t, err) + require.NotEmpty(t, obtainedEVs) + require.Len(t, obtainedEVs, 1) + + obtainedEvB := obtainedEVs[0] + require.Equal(t, createdEvB.Namespace, obtainedEvB.Namespace) + require.Equal(t, createdEvB.Name, obtainedEvB.Name) + require.Equal(t, createdEvB.EncryptedData, obtainedEvB.EncryptedData) + + // List all encrypted values, until a certain time + pastTime := time.Now().Add(-1 * time.Hour).Unix() + obtainedEVs, err = sut.GlobalEncryptedValueStorage.ListAll(t.Context(), contracts.ListOpts{}, &pastTime) + require.NoError(t, err) + require.Empty(t, obtainedEVs) + }) + + t.Run("counting encrypted values returns their total", func(t *testing.T) { + t.Parallel() + + sut := testutils.Setup(t) + _, err := sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-a", "test-name", 1, []byte("test-data")) + require.NoError(t, err) + + _, err = sut.EncryptedValueStorage.Create(t.Context(), "test-namespace-b", "test-name", 1, []byte("test-data")) + require.NoError(t, err) + + count, err := sut.GlobalEncryptedValueStorage.CountAll(t.Context(), nil) + require.NoError(t, err) + require.Equal(t, int64(2), count) + + // Count all encrypted values, until a certain time + pastTime := time.Now().Add(-1 * time.Hour).Unix() + count, err = sut.GlobalEncryptedValueStorage.CountAll(t.Context(), &pastTime) + require.NoError(t, err) + require.Equal(t, int64(0), count) + }) } func TestStateMachine(t *testing.T) { diff --git a/pkg/storage/secret/encryption/query.go b/pkg/storage/secret/encryption/query.go index 573e295dcaa..47ea83ce0cd 100644 --- a/pkg/storage/secret/encryption/query.go +++ b/pkg/storage/secret/encryption/query.go @@ -17,10 +17,12 @@ var ( sqlTemplates = template.Must(template.New("sql").ParseFS(sqlTemplatesFS, `data/*.sql`)) // The SQL Commands - sqlEncryptedValueCreate = mustTemplate("encrypted_value_create.sql") - sqlEncryptedValueRead = mustTemplate("encrypted_value_read.sql") - sqlEncryptedValueUpdate = mustTemplate("encrypted_value_update.sql") - sqlEncryptedValueDelete = mustTemplate("encrypted_value_delete.sql") + sqlEncryptedValueCreate = mustTemplate("encrypted_value_create.sql") + sqlEncryptedValueRead = mustTemplate("encrypted_value_read.sql") + sqlEncryptedValueUpdate = mustTemplate("encrypted_value_update.sql") + sqlEncryptedValueDelete = mustTemplate("encrypted_value_delete.sql") + sqlEncryptedValueListAll = mustTemplate("encrypted_value_list_all.sql") + sqlEncryptedValueCountAll = mustTemplate("encrypted_value_count_all.sql") sqlDataKeyCreate = mustTemplate("data_key_create.sql") sqlDataKeyRead = mustTemplate("data_key_read.sql") @@ -93,6 +95,24 @@ func (r deleteEncryptedValue) Validate() error { return nil // TODO } +type listAllEncryptedValues struct { + sqltemplate.SQLTemplate + Limit int64 + Offset int64 + HasUntilTime bool + UntilTime int64 +} + +func (r listAllEncryptedValues) Validate() error { return nil } + +type countAllEncryptedValues struct { + sqltemplate.SQLTemplate + HasUntilTime bool + UntilTime int64 +} + +func (r countAllEncryptedValues) Validate() error { return nil } + /*************************************/ /**-- Data Key Queries --**/ /*************************************/ diff --git a/pkg/storage/secret/encryption/query_test.go b/pkg/storage/secret/encryption/query_test.go index c2ebf7a9635..a93ff0b77e6 100644 --- a/pkg/storage/secret/encryption/query_test.go +++ b/pkg/storage/secret/encryption/query_test.go @@ -10,6 +10,7 @@ import ( ) func TestEncryptedValueQueries(t *testing.T) { + untilTime := int64(1234) mocks.CheckQuerySnapshots(t, mocks.TemplateTestSetup{ RootDir: "testdata", Templates: map[*template.Template][]mocks.TemplateTestCase{ @@ -64,6 +65,63 @@ func TestEncryptedValueQueries(t *testing.T) { }, }, }, + sqlEncryptedValueListAll: { + { + Name: "list_limit_10_offset_0", + Data: &listAllEncryptedValues{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Limit: 10, + Offset: 0, + HasUntilTime: false, + }, + }, + { + Name: "list_limit_10_offset_2", + Data: &listAllEncryptedValues{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Limit: 10, + Offset: 2, + HasUntilTime: false, + }, + }, + { + Name: "list_all", + Data: &listAllEncryptedValues{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Limit: 0, + Offset: 0, + HasUntilTime: false, + }, + }, + { + Name: "list_all_until_time", + Data: &listAllEncryptedValues{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + Limit: 0, + Offset: 0, + HasUntilTime: true, + UntilTime: untilTime, + }, + }, + }, + sqlEncryptedValueCountAll: { + { + Name: "count_all", + Data: &countAllEncryptedValues{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + HasUntilTime: false, + UntilTime: 0, + }, + }, + { + Name: "count_all_until_time", + Data: &countAllEncryptedValues{ + SQLTemplate: mocks.NewTestingSQLTemplate(), + HasUntilTime: true, + UntilTime: untilTime, + }, + }, + }, }, }) } diff --git a/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all.sql b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all.sql new file mode 100755 index 00000000000..fc8594ad307 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all.sql @@ -0,0 +1,4 @@ +SELECT COUNT(*) AS count +FROM + `secret_encrypted_value` +; diff --git a/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all_until_time.sql b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all_until_time.sql new file mode 100755 index 00000000000..760bc4432cc --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_count_all-count_all_until_time.sql @@ -0,0 +1,5 @@ +SELECT COUNT(*) AS count +FROM + `secret_encrypted_value` +WHERE `created` <= 1234 +; diff --git a/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all.sql b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all.sql new file mode 100755 index 00000000000..74b699d45f1 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all.sql @@ -0,0 +1,11 @@ +SELECT + `namespace`, + `name`, + `version`, + `encrypted_data`, + `created`, + `updated` +FROM + `secret_encrypted_value` +ORDER BY `created` ASC +; diff --git a/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all_until_time.sql b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all_until_time.sql new file mode 100755 index 00000000000..b34496aaf93 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_all_until_time.sql @@ -0,0 +1,12 @@ +SELECT + `namespace`, + `name`, + `version`, + `encrypted_data`, + `created`, + `updated` +FROM + `secret_encrypted_value` +WHERE `created` <= 1234 +ORDER BY `created` ASC +; diff --git a/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_0.sql b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_0.sql new file mode 100755 index 00000000000..9c33fd6bdbf --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_0.sql @@ -0,0 +1,12 @@ +SELECT + `namespace`, + `name`, + `version`, + `encrypted_data`, + `created`, + `updated` +FROM + `secret_encrypted_value` +ORDER BY `created` ASC +LIMIT 10 OFFSET 0 +; diff --git a/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_2.sql b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_2.sql new file mode 100755 index 00000000000..d7066395a78 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/mysql--encrypted_value_list_all-list_limit_10_offset_2.sql @@ -0,0 +1,12 @@ +SELECT + `namespace`, + `name`, + `version`, + `encrypted_data`, + `created`, + `updated` +FROM + `secret_encrypted_value` +ORDER BY `created` ASC +LIMIT 10 OFFSET 2 +; diff --git a/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all.sql b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all.sql new file mode 100755 index 00000000000..91c725a708f --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all.sql @@ -0,0 +1,4 @@ +SELECT COUNT(*) AS count +FROM + "secret_encrypted_value" +; diff --git a/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all_until_time.sql b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all_until_time.sql new file mode 100755 index 00000000000..1c691378494 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_count_all-count_all_until_time.sql @@ -0,0 +1,5 @@ +SELECT COUNT(*) AS count +FROM + "secret_encrypted_value" +WHERE "created" <= 1234 +; diff --git a/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all.sql b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all.sql new file mode 100755 index 00000000000..74432ebbf69 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all.sql @@ -0,0 +1,11 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +ORDER BY "created" ASC +; diff --git a/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all_until_time.sql b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all_until_time.sql new file mode 100755 index 00000000000..1d7089f751e --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_all_until_time.sql @@ -0,0 +1,12 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +WHERE "created" <= 1234 +ORDER BY "created" ASC +; diff --git a/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_0.sql b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_0.sql new file mode 100755 index 00000000000..6f2bbd0b90f --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_0.sql @@ -0,0 +1,12 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +ORDER BY "created" ASC +LIMIT 10 OFFSET 0 +; diff --git a/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_2.sql b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_2.sql new file mode 100755 index 00000000000..b9f326c8bf0 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/postgres--encrypted_value_list_all-list_limit_10_offset_2.sql @@ -0,0 +1,12 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +ORDER BY "created" ASC +LIMIT 10 OFFSET 2 +; diff --git a/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all.sql b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all.sql new file mode 100755 index 00000000000..91c725a708f --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all.sql @@ -0,0 +1,4 @@ +SELECT COUNT(*) AS count +FROM + "secret_encrypted_value" +; diff --git a/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all_until_time.sql b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all_until_time.sql new file mode 100755 index 00000000000..1c691378494 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_count_all-count_all_until_time.sql @@ -0,0 +1,5 @@ +SELECT COUNT(*) AS count +FROM + "secret_encrypted_value" +WHERE "created" <= 1234 +; diff --git a/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all.sql b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all.sql new file mode 100755 index 00000000000..74432ebbf69 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all.sql @@ -0,0 +1,11 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +ORDER BY "created" ASC +; diff --git a/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all_until_time.sql b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all_until_time.sql new file mode 100755 index 00000000000..1d7089f751e --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_all_until_time.sql @@ -0,0 +1,12 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +WHERE "created" <= 1234 +ORDER BY "created" ASC +; diff --git a/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_0.sql b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_0.sql new file mode 100755 index 00000000000..6f2bbd0b90f --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_0.sql @@ -0,0 +1,12 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +ORDER BY "created" ASC +LIMIT 10 OFFSET 0 +; diff --git a/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_2.sql b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_2.sql new file mode 100755 index 00000000000..b9f326c8bf0 --- /dev/null +++ b/pkg/storage/secret/encryption/testdata/sqlite--encrypted_value_list_all-list_limit_10_offset_2.sql @@ -0,0 +1,12 @@ +SELECT + "namespace", + "name", + "version", + "encrypted_data", + "created", + "updated" +FROM + "secret_encrypted_value" +ORDER BY "created" ASC +LIMIT 10 OFFSET 2 +; From 4da2c50990cf2fd16e97a52be082a8e254f816c5 Mon Sep 17 00:00:00 2001 From: ti361 Date: Mon, 28 Jul 2025 18:10:20 +0800 Subject: [PATCH 2/4] change method "GET" to "DELETE" with the "Delete a report" API (#108750) --- docs/sources/developers/http_api/reporting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/developers/http_api/reporting.md b/docs/sources/developers/http_api/reporting.md index 8666f253e63..4b02276975b 100644 --- a/docs/sources/developers/http_api/reporting.md +++ b/docs/sources/developers/http_api/reporting.md @@ -401,7 +401,7 @@ See note in the [introduction](#reporting-api) for an explanation. ### Example request ```http -GET /api/reports/6 HTTP/1.1 +DELETE /api/reports/6 HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk From af376cd286a59eeca7870d4c373bd438845554c7 Mon Sep 17 00:00:00 2001 From: Konrad Lalik Date: Mon, 28 Jul 2025 12:22:36 +0200 Subject: [PATCH 3/4] Alerting: Fix backtick handling when hashing queries (#108757) Update the hashQuery algorithm to handle more formatting edge cases --- .../utils/__snapshots__/rule-id.test.tsx.snap | 4 +- .../alerting/unified/utils/rule-id.test.tsx | 118 +++++++++++++++++- .../alerting/unified/utils/rule-id.ts | 14 ++- 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/public/app/features/alerting/unified/utils/__snapshots__/rule-id.test.tsx.snap b/public/app/features/alerting/unified/utils/__snapshots__/rule-id.test.tsx.snap index 00fe79eb53d..9facadeeacc 100644 --- a/public/app/features/alerting/unified/utils/__snapshots__/rule-id.test.tsx.snap +++ b/public/app/features/alerting/unified/utils/__snapshots__/rule-id.test.tsx.snap @@ -1,5 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`hashRulerRule should hash alerting rule 1`] = `"7317348"`; +exports[`hashRulerRule should hash alerting rule 1`] = `"852037155"`; -exports[`hashRulerRule should hash recording rules 1`] = `"-447747460"`; +exports[`hashRulerRule should hash recording rules 1`] = `"914562864"`; diff --git a/public/app/features/alerting/unified/utils/rule-id.test.tsx b/public/app/features/alerting/unified/utils/rule-id.test.tsx index e2652c11885..7fedf534b3b 100644 --- a/public/app/features/alerting/unified/utils/rule-id.test.tsx +++ b/public/app/features/alerting/unified/utils/rule-id.test.tsx @@ -13,7 +13,15 @@ import { RulerRecordingRuleDTO, } from 'app/types/unified-alerting-dto'; -import { equal, getRuleIdFromPathname, hashRule, hashRulerRule, parse, stringifyIdentifier } from './rule-id'; +import { + equal, + getRuleIdFromPathname, + hashQuery, + hashRule, + hashRulerRule, + parse, + stringifyIdentifier, +} from './rule-id'; const alertingRule = { prom: { @@ -258,3 +266,111 @@ describe('useRuleIdFromPathname', () => { expect(result.current).toBe('abc%25def'); }); }); + +describe('hashQuery', () => { + it('should produce the same hash for queries with different whitespace formatting', () => { + const query1 = `sum by (client,origin,destination,met_val)( + sum_over_time( + {client=~"PRU|RVSI"} + ) +)`; + const query2 = `sum by (client,origin,destination,met_val)(sum_over_time({client=~"PRU|RVSI"}))`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should produce the same hash for queries with and without outer parentheses', () => { + const query1 = `sum by (client)(rate(requests_total[5m]))`; + const query2 = `(sum by (client)(rate(requests_total[5m])))`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should produce the same hash for queries with different quote types in label formats', () => { + const query1 = `label_format origin=\`{{.app_host}}\``; + const query2 = `label_format origin="{{.app_host}}"`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should produce the same hash for queries with escaped vs unescaped quotes', () => { + const query1 = `label_format met_val=\`{{"REQ_SENT"}}\``; + const query2 = `label_format met_val="{{\"REQ_SENT\"}}"`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should handle complex Loki recording rules with all formatting differences', () => { + const query1 = `sum by (client,origin,destination,metric_type)( + sum_over_time( + {client=~"FOO|BAR|BAZ", service_name="app_sessions"} + |= "server" + |= "component" + | logfmt + | label_format origin=\`{{.host_name}}\` + | label_format destination=\`{{.component_name}}\` + | label_format metric_type=\`{{"REQUEST_COUNT"}}\` + | keep client,destination,origin,metric_type,response_time + | unwrap response_time + [5m]) +) > 0`; + + const query2 = `(sum by (client,origin,destination,metric_type)(sum_over_time({client=~"FOO|BAR|BAZ", service_name="app_sessions"} |= "server" |= "component" | logfmt | label_format origin="{{.host_name}}" | label_format destination="{{.component_name}}" | label_format metric_type="{{\"REQUEST_COUNT\"}}" | keep client,destination,origin,metric_type,response_time | unwrap response_time[5m])) > 0)`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should produce the same hash for queries with reordered label matchers', () => { + const query1 = `{job="prometheus", instance="localhost:9090"}`; + const query2 = `{instance="localhost:9090", job="prometheus"}`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should handle multiple types of brackets and quotes', () => { + const query1 = `rate(http_requests_total{method="GET"}[5m])`; + const query2 = `rate(http_requests_total{method=\`GET\`}[5m])`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should normalize backslashes properly', () => { + const query1 = `label_format path="{{.file_path}}"`; + const query2 = `label_format path="{{\.file_path}}"`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should handle empty queries', () => { + expect(hashQuery('')).toBe(''); + }); + + it('should handle queries with only parentheses', () => { + expect(hashQuery('()')).toBe(''); + }); + + it('should handle complex nested parentheses and brackets', () => { + const query1 = `((sum(rate(requests[5m]))))`; + const query2 = `sum(rate(requests[5m]))`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should handle mixed quote escaping scenarios', () => { + const query1 = `label_format msg=\`{{"error: \\"timeout\\""}}\``; + const query2 = `label_format msg="{{\"error: \\\"timeout\\\"\"}}"`; + + expect(hashQuery(query1)).toBe(hashQuery(query2)); + }); + + it('should produce consistent results for character sorting', () => { + const query1 = `abc{x="1",y="2"}`; + const query2 = `abc{y="2",x="1"}`; + + const hash1 = hashQuery(query1); + const hash2 = hashQuery(query2); + + expect(hash1).toBe(hash2); + expect(hash1).toBe(hash1.split('').sort().join('')); + }); +}); diff --git a/public/app/features/alerting/unified/utils/rule-id.ts b/public/app/features/alerting/unified/utils/rule-id.ts index b2eb9a663c0..f8db7346df3 100644 --- a/public/app/features/alerting/unified/utils/rule-id.ts +++ b/public/app/features/alerting/unified/utils/rule-id.ts @@ -311,9 +311,21 @@ export function hashQuery(query: string) { if (query.length > 1 && query[0] === '(' && query[query.length - 1] === ')') { query = query.slice(1, -1); } + // whitespace could be added or removed query = query.replace(/\s|\n/g, ''); - // labels matchers can be reordered, so sort the enitre string, esentially comparing just the character counts + + // normalize escaped quotes in template strings like {{\"REQ_SENT\"}} -> {{"REQ_SENT"}} + query = query.replace(/\\"/g, '"'); + + // normalize backtick template strings to double quotes for consistency + // Convert `{{.field}}` to "{{.field}}" + query = query.replace(/`([^`]*)`/g, '"$1"'); + + // remove quotes, brackets, parentheses, backslashes, and backticks + query = query.replace(/['"()\[\]\\`]/g, ''); + + // labels matchers can be reordered, so sort the entire string, essentially comparing just the character counts return query.split('').sort().join(''); } From 814fccb970515f4a7738339a99f7832726d54c2a Mon Sep 17 00:00:00 2001 From: Gilles De Mey Date: Mon, 28 Jul 2025 12:31:51 +0200 Subject: [PATCH 4/4] Alerting: Use the official maildev image again (#108768) --- devenv/docker/blocks/maildev/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devenv/docker/blocks/maildev/docker-compose.yaml b/devenv/docker/blocks/maildev/docker-compose.yaml index 7cc648a23d1..50d639e13bc 100644 --- a/devenv/docker/blocks/maildev/docker-compose.yaml +++ b/devenv/docker/blocks/maildev/docker-compose.yaml @@ -1,5 +1,5 @@ maildev: - image: gillesdemey/maildev + image: maildev/maildev:2.2.1 ports: - "12080:1080" - "1025:1025"