Secrets: garbage collection (#110247)

* clean up older secret versions

* start gargbage collection worker as background service

* make gen-go

* fix typo

* make update-workspace

* undo go mod changes

* undo go work sum changes

* Update pkg/registry/apis/secret/garbagecollectionworker/worker.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* Update pkg/registry/apis/secret/garbagecollectionworker/worker.go

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

* default gc_worker_batch_size to 1 minute

* fix typo

* fix typo

* add test to ensure cleaning up secure values is idempotent

* make gen-go

* make update-workspace

* undo go.mod and .sum changes

* undo enterprise imports

---------

Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

pkg/storage/secret/metadata/data/secure_value_delete.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  {{ .Ident "secret_secure_value" }}
+WHERE 
+  {{ .Ident "namespace" }} = {{ .Arg .Namespace }} AND
+  {{ .Ident "name" }} = {{ .Arg .Name }} AND
+  {{ .Ident "version" }} = {{ .Arg .Version }}
+;

pkg/storage/secret/metadata/data/secure_value_lease_inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  {{ .Ident "secret_secure_value" }}
+SET
+  {{ .Ident "lease_token" }} = {{ .Arg .LeaseToken }},
+  {{ .Ident "lease_created" }} = {{ .Arg .Now }}
+WHERE 
+  {{ .Ident "guid" }} IN (SELECT {{ .Ident "guid"}} 
+                          FROM {{ .Ident "secret_secure_value" }} 
+                          WHERE 
+                            {{ .Ident "active" }} = FALSE AND
+                            {{ .Arg .Now }} - {{ .Ident "created" }} > {{ .Arg .MinAge }} AND
+                            {{ .Arg .Now }} - {{ .Ident "lease_created" }} > {{ .Arg .LeaseTTL }}
+                          LIMIT {{ .Arg .MaxBatchSize }})
+;

pkg/storage/secret/metadata/data/secure_value_list_by_lease_token.sql

@@ -0,0 +1,26 @@
+SELECT
+  {{ .Ident "guid" }},
+  {{ .Ident "name" }},
+  {{ .Ident "namespace" }},
+  {{ .Ident "annotations" }},
+  {{ .Ident "labels" }},
+  {{ .Ident "created" }},
+  {{ .Ident "created_by" }},
+  {{ .Ident "updated" }},
+  {{ .Ident "updated_by" }},
+  {{ .Ident "description" }},
+  {{ .Ident "keeper" }},
+  {{ .Ident "decrypters" }},
+  {{ .Ident "ref" }},
+  {{ .Ident "external_id" }},
+  {{ .Ident "version" }},
+  {{ .Ident "active" }},
+  {{ .Ident "owner_reference_api_group" }},
+  {{ .Ident "owner_reference_api_version" }},
+  {{ .Ident "owner_reference_kind" }},
+  {{ .Ident "owner_reference_name" }}
+FROM
+  {{ .Ident "secret_secure_value" }}
+WHERE 
+  {{ .Ident "lease_token" }} = {{ .Arg .LeaseToken }}
+;

pkg/storage/secret/metadata/metrics/metrics.go

@@ -30,6 +30,7 @@ type StorageMetrics struct {
 	SecureValueMetadataListDuration   *prometheus.HistogramVec
 	SecureValueSetExternalIDDuration  *prometheus.HistogramVec
 	SecureValueSetStatusDuration      *prometheus.HistogramVec
+	SecureValueDeleteDuration         *prometheus.HistogramVec
 
 	DecryptDuration *prometheus.HistogramVec
 }
@@ -116,6 +117,13 @@ func newStorageMetrics() *StorageMetrics {
 			Help:      "Duration of secure value set status operations",
 			Buckets:   prometheus.DefBuckets,
 		}, []string{successLabel}),
+		SecureValueDeleteDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "secure_value_delete_duration_seconds",
+			Help:      "Duration of secure value delete operations",
+			Buckets:   prometheus.DefBuckets,
+		}, []string{successLabel}),
 
 		// Decrypt metrics
 		DecryptDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
@@ -151,6 +159,7 @@ func NewStorageMetrics(reg prometheus.Registerer) *StorageMetrics {
 				m.SecureValueMetadataListDuration,
 				m.SecureValueSetExternalIDDuration,
 				m.SecureValueSetStatusDuration,
+				m.SecureValueDeleteDuration,
 				m.DecryptDuration,
 			)
 		}

pkg/storage/secret/metadata/query.go

@@ -28,6 +28,9 @@ var (
 	sqlSecureValueList             = mustTemplate("secure_value_list.sql")
 	sqlSecureValueCreate           = mustTemplate("secure_value_create.sql")
 	sqlSecureValueUpdateExternalId = mustTemplate("secure_value_updateExternalId.sql")
+	sqlSecureValueDelete           = mustTemplate("secure_value_delete.sql")
+	sqlSecureValueLeaseInactive    = mustTemplate("secure_value_lease_inactive.sql")
+	sqlSecureValueListByLeaseToken = mustTemplate("secure_value_list_by_lease_token.sql")
 
 	sqlGetLatestSecureValueVersion     = mustTemplate("secure_value_get_latest_version.sql")
 	sqlSecureValueSetVersionToActive   = mustTemplate("secure_value_set_version_to_active.sql")
@@ -208,3 +211,39 @@ type updateExternalIdSecureValue struct {
 func (r updateExternalIdSecureValue) Validate() error {
 	return nil // TODO
 }
+
+type deleteSecureValue struct {
+	sqltemplate.SQLTemplate
+	Namespace string
+	Name      string
+	Version   int64
+}
+
+// Validate is only used if we use `dbutil` from `unifiedstorage`
+func (r deleteSecureValue) Validate() error {
+	return nil // TODO
+}
+
+type leaseInactiveSecureValues struct {
+	sqltemplate.SQLTemplate
+	LeaseToken   string
+	MaxBatchSize uint16
+	MinAge       int64
+	LeaseTTL     int64
+	Now          int64
+}
+
+// Validate is only used if we use `dbutil` from `unifiedstorage`
+func (r leaseInactiveSecureValues) Validate() error {
+	return nil // TODO
+}
+
+type listSecureValuesByLeaseToken struct {
+	sqltemplate.SQLTemplate
+	LeaseToken string
+}
+
+// Validate is only used if we use `dbutil` from `unifiedstorage`
+func (r listSecureValuesByLeaseToken) Validate() error {
+	return nil // TODO
+}

pkg/storage/secret/metadata/query_test.go

@@ -3,6 +3,7 @@ package metadata
 import (
 	"testing"
 	"text/template"
+	"time"
 
 	"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
 	"k8s.io/utils/ptr"
@@ -238,6 +239,39 @@ func TestSecureValueQueries(t *testing.T) {
 					},
 				},
 			},
+			sqlSecureValueDelete: {
+				{
+					Name: "deleteSecureValue",
+					Data: &deleteSecureValue{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Namespace:   "ns",
+						Name:        "name",
+						Version:     1,
+					},
+				},
+			},
+			sqlSecureValueLeaseInactive: {
+				{
+					Name: "lease inactive",
+					Data: &leaseInactiveSecureValues{
+						SQLTemplate:  mocks.NewTestingSQLTemplate(),
+						Now:          10,
+						LeaseToken:   "token",
+						LeaseTTL:     int64((30 * time.Second).Seconds()),
+						MaxBatchSize: 10,
+						MinAge:       int64((300 * time.Second).Seconds()),
+					},
+				},
+			},
+			sqlSecureValueListByLeaseToken: {
+				{
+					Name: "list by lease token",
+					Data: &listSecureValuesByLeaseToken{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						LeaseToken:  "token",
+					},
+				},
+			},
 		},
 	})
 }

pkg/storage/secret/metadata/secure_value_model.go

@@ -122,18 +122,18 @@ func (sv *secureValueDB) toKubernetes() (*secretv1beta1.SecureValue, error) {
 }
 
 // toCreateRow maps a Kubernetes resource into a DB row for new resources being created/inserted.
-func toCreateRow(sv *secretv1beta1.SecureValue, actorUID string) (*secureValueDB, error) {
+func toCreateRow(now time.Time, sv *secretv1beta1.SecureValue, actorUID string) (*secureValueDB, error) {
 	row, err := toRow(sv, "")
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert SecureValue to secureValueDB: %w", err)
 	}
 
-	now := time.Now().UTC().Unix()
+	timestamp := now.UTC().Unix()
 
 	row.GUID = uuid.New().String()
-	row.Created = now
+	row.Created = timestamp
 	row.CreatedBy = actorUID
-	row.Updated = now
+	row.Updated = timestamp
 	row.UpdatedBy = actorUID
 
 	return row, nil

pkg/storage/secret/metadata/secure_value_store.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -23,11 +24,13 @@ import (
 var _ contracts.SecureValueMetadataStorage = (*secureValueMetadataStorage)(nil)
 
 func ProvideSecureValueMetadataStorage(
+	clock contracts.Clock,
 	db contracts.Database,
 	tracer trace.Tracer,
 	reg prometheus.Registerer,
 ) (contracts.SecureValueMetadataStorage, error) {
 	return &secureValueMetadataStorage{
+		clock:   clock,
 		db:      db,
 		dialect: sqltemplate.DialectForDriver(db.DriverName()),
 		metrics: metrics.NewStorageMetrics(reg),
@@ -37,6 +40,7 @@ func ProvideSecureValueMetadataStorage(
 
 // secureValueMetadataStorage is the actual implementation of the secure value (metadata) storage.
 type secureValueMetadataStorage struct {
+	clock   contracts.Clock
 	db      contracts.Database
 	dialect sqltemplate.Dialect
 	metrics *metrics.StorageMetrics
@@ -44,7 +48,7 @@ type secureValueMetadataStorage struct {
 }
 
 func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv1beta1.SecureValue, actorUID string) (_ *secretv1beta1.SecureValue, svmCreateErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	name := sv.GetName()
 	namespace := sv.GetNamespace()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Create", trace.WithAttributes(
@@ -123,7 +127,7 @@ func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv1bet
 		for {
 			sv.Status.Version = version
 
-			row, err = toCreateRow(sv, actorUID)
+			row, err = toCreateRow(s.clock.Now(), sv, actorUID)
 			if err != nil {
 				return fmt.Errorf("to create row: %w", err)
 			}
@@ -256,7 +260,7 @@ func (s *secureValueMetadataStorage) readActiveVersion(ctx context.Context, name
 }
 
 func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.Namespace, name string, opts contracts.ReadOpts) (_ *secretv1beta1.SecureValue, readErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Read", trace.WithAttributes(
 		attribute.String("name", name),
 		attribute.String("namespace", namespace.String()),
@@ -297,7 +301,7 @@ func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.N
 }
 
 func (s *secureValueMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) (svList []secretv1beta1.SecureValue, listErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.List", trace.WithAttributes(
 		attribute.String("namespace", namespace.String()),
 	))
@@ -400,7 +404,7 @@ func (s *secureValueMetadataStorage) SetVersionToActive(ctx context.Context, nam
 		return fmt.Errorf("setting secure value version to active: namespace=%+v name=%+v version=%+v %w", namespace, name, version, err)
 	}
 
-	// validate modified cound
+	// validate modified count
 	modifiedCount, err := res.RowsAffected()
 	if err != nil {
 		return fmt.Errorf("fetching number of modified rows: %w", err)
@@ -449,7 +453,7 @@ func (s *secureValueMetadataStorage) SetVersionToInactive(ctx context.Context, n
 }
 
 func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespace xkube.Namespace, name string, version int64, externalID contracts.ExternalID) (setExtIDErr error) {
-	start := time.Now()
+	start := s.clock.Now()
 	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.SetExternalID", trace.WithAttributes(
 		attribute.String("name", name),
 		attribute.String("namespace", namespace.String()),
@@ -508,3 +512,164 @@ func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespac
 
 	return nil
 }
+
+func (s *secureValueMetadataStorage) Delete(ctx context.Context, namespace xkube.Namespace, name string, version int64) (err error) {
+	start := s.clock.Now()
+	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Delete", trace.WithAttributes(
+		attribute.String("name", name),
+		attribute.String("namespace", namespace.String()),
+		attribute.Int64("version", version),
+	))
+
+	defer span.End()
+
+	defer func() {
+		success := err == nil
+		args := []any{
+			"namespace", namespace.String(),
+			"name", name,
+			"version", strconv.FormatInt(version, 10),
+			"success", success,
+		}
+
+		if !success {
+			span.SetStatus(codes.Error, "SecureValueMetadataStorage.Delete failed")
+			span.RecordError(err)
+			args = append(args, "error", err)
+		}
+
+		logging.FromContext(ctx).Info("SecureValueMetadataStorage.Delete", args...)
+		s.metrics.SecureValueDeleteDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
+	}()
+
+	req := deleteSecureValue{
+		SQLTemplate: sqltemplate.New(s.dialect),
+		Namespace:   namespace.String(),
+		Name:        name,
+		Version:     version,
+	}
+
+	q, err := sqltemplate.Execute(sqlSecureValueDelete, req)
+	if err != nil {
+		return fmt.Errorf("execute template %q: %w", sqlSecureValueDelete.Name(), err)
+	}
+
+	res, err := s.db.ExecContext(ctx, q, req.GetArgs()...)
+	if err != nil {
+		return fmt.Errorf("deleting secure value: namespace=%+v name=%+v version=%+v %w", namespace, name, version, err)
+	}
+
+	modifiedCount, err := res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("getting rows affected: %w", err)
+	}
+	// Deleting is idempotent so modifiedCunt must be in {0, 1}
+	if modifiedCount > 1 {
+		return fmt.Errorf("secureValueMetadataStorage.Delete: delete more than one secret, this is a bug, check the where condition: modifiedCount=%d", modifiedCount)
+	}
+
+	return nil
+}
+
+func (s *secureValueMetadataStorage) LeaseInactiveSecureValues(ctx context.Context, maxBatchSize uint16) (out []secretv1beta1.SecureValue, err error) {
+	start := s.clock.Now()
+	ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.LeaseInactiveSecureValues", trace.WithAttributes(
+		attribute.Int("maxBatchSize", int(maxBatchSize)),
+	))
+
+	defer span.End()
+
+	defer func() {
+		success := err == nil
+
+		if !success {
+			span.SetStatus(codes.Error, "SecureValueMetadataStorage.LeaseInactiveSecureValues failed")
+			span.RecordError(err)
+		}
+
+		s.metrics.SecureValueDeleteDuration.WithLabelValues(strconv.FormatBool(success)).Observe(time.Since(start).Seconds())
+	}()
+
+	leaseToken := uuid.NewString()
+	if err := s.acquireLeases(ctx, leaseToken, maxBatchSize); err != nil {
+		return nil, fmt.Errorf("acquiring leases for inactive secure values: %w", err)
+	}
+
+	secureValues, err := s.listByLeaseToken(ctx, leaseToken)
+	if err != nil {
+		return nil, fmt.Errorf("fetching secure values by lease token: %w", err)
+	}
+
+	return secureValues, nil
+}
+
+func (s *secureValueMetadataStorage) acquireLeases(ctx context.Context, leaseToken string, maxBatchSize uint16) error {
+	req := leaseInactiveSecureValues{
+		SQLTemplate:  sqltemplate.New(s.dialect),
+		LeaseToken:   leaseToken,
+		MaxBatchSize: maxBatchSize,
+		MinAge:       int64((300 * time.Second).Seconds()),
+		LeaseTTL:     int64((30 * time.Second).Seconds()),
+		Now:          s.clock.Now().UTC().Unix(),
+	}
+
+	q, err := sqltemplate.Execute(sqlSecureValueLeaseInactive, req)
+	if err != nil {
+		return fmt.Errorf("execute template %q: %w", sqlSecureValueLeaseInactive.Name(), err)
+	}
+
+	if _, err := s.db.ExecContext(ctx, q, req.GetArgs()...); err != nil {
+		return fmt.Errorf("leasing inactive secure values: %w", err)
+	}
+
+	return nil
+}
+
+func (s *secureValueMetadataStorage) listByLeaseToken(ctx context.Context, leaseToken string) ([]secretv1beta1.SecureValue, error) {
+	req := listSecureValuesByLeaseToken{
+		SQLTemplate: sqltemplate.New(s.dialect),
+		LeaseToken:  leaseToken,
+	}
+
+	q, err := sqltemplate.Execute(sqlSecureValueListByLeaseToken, req)
+	if err != nil {
+		return nil, fmt.Errorf("execute template %q: %w", sqlSecureValueListByLeaseToken.Name(), err)
+	}
+
+	rows, err := s.db.QueryContext(ctx, q, req.GetArgs()...)
+	if err != nil {
+		return nil, fmt.Errorf("listing secure values: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	secureValues := make([]secretv1beta1.SecureValue, 0)
+	for rows.Next() {
+		row := secureValueDB{}
+
+		err = rows.Scan(&row.GUID,
+			&row.Name, &row.Namespace, &row.Annotations,
+			&row.Labels,
+			&row.Created, &row.CreatedBy,
+			&row.Updated, &row.UpdatedBy,
+			&row.Description, &row.Keeper, &row.Decrypters,
+			&row.Ref, &row.ExternalID, &row.Version, &row.Active,
+			&row.OwnerReferenceAPIGroup, &row.OwnerReferenceAPIVersion, &row.OwnerReferenceKind, &row.OwnerReferenceName,
+		)
+
+		if err != nil {
+			return nil, fmt.Errorf("error reading secure value row: %w", err)
+		}
+
+		secureValue, err := row.toKubernetes()
+		if err != nil {
+			return nil, fmt.Errorf("convert to kubernetes object: %w", err)
+		}
+
+		secureValues = append(secureValues, *secureValue)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("read rows error: %w", err)
+	}
+
+	return secureValues, nil
+}

pkg/storage/secret/metadata/secure_value_store_test.go

@@ -3,13 +3,17 @@ package metadata_test
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel/trace/noop"
 	"k8s.io/utils/ptr"
+	"pgregory.net/rapid"
 
 	secretv1beta1 "github.com/grafana/grafana/apps/secret/pkg/apis/secret/v1beta1"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/clock"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
+	"github.com/grafana/grafana/pkg/registry/apis/secret/testutils"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/storage/secret/database"
@@ -43,7 +47,7 @@ func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
 	db := database.ProvideDatabase(testDB, tracer)
 
 	// Initialize the secure value storage
-	secureValueStorage, err := metadata.ProvideSecureValueMetadataStorage(db, tracer, nil)
+	secureValueStorage, err := metadata.ProvideSecureValueMetadataStorage(clock.ProvideClock(), db, tracer, nil)
 	require.NoError(t, err)
 
 	// Initialize the keeper storage
@@ -142,3 +146,96 @@ func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
 		require.Equal(t, contracts.ErrSecureValueNotFound, err)
 	})
 }
+
+func TestLeaseInactiveSecureValues(t *testing.T) {
+	t.Parallel()
+
+	t.Run("no secure value exists", func(t *testing.T) {
+		t.Parallel()
+
+		sut := testutils.Setup(t)
+		svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		require.Empty(t, svs)
+	})
+
+	t.Run("secure values are not visible to other requests during lease duration", func(t *testing.T) {
+		sut := testutils.Setup(t)
+		sv, err := sut.CreateSv(t.Context())
+		require.NoError(t, err)
+		_, err = sut.DeleteSv(t.Context(), sv.Namespace, sv.Name)
+		require.NoError(t, err)
+		// Advance clock to handle grace period
+		sut.Clock.AdvanceBy(10 * time.Minute)
+		// Acquire a lease on inactive secure values
+		values1, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(values1))
+		require.Equal(t, sv.UID, values1[0].UID)
+		// Try to acquire a lease again
+		values2, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		// There's only one inactive secure value and it is already leased
+		require.Empty(t, values2)
+		// Advance clock to expire lease
+		sut.Clock.AdvanceBy(10 * time.Minute)
+		values3, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), 10)
+		require.NoError(t, err)
+		// Should acquire a new lease since the previous one expired
+		require.Equal(t, 1, len(values3))
+		require.Equal(t, sv.UID, values3[0].UID)
+	})
+}
+
+func TestPropertySecureValueMetadataStorage(t *testing.T) {
+	t.Parallel()
+
+	tt := t
+
+	rapid.Check(t, func(t *rapid.T) {
+		sut := testutils.Setup(tt)
+		model := newModel()
+
+		t.Repeat(map[string]func(*rapid.T){
+			"create": func(t *rapid.T) {
+				sv := anySecureValueGen.Draw(t, "sv")
+				modelCreatedSv, modelErr := model.create(sut.Clock.Now(), deepCopy(sv))
+				createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(deepCopy(sv)))
+				if err != nil || modelErr != nil {
+					require.ErrorIs(t, err, modelErr)
+					return
+				}
+				require.Equal(t, modelCreatedSv.Namespace, createdSv.Namespace)
+				require.Equal(t, modelCreatedSv.Name, createdSv.Name)
+				require.Equal(t, modelCreatedSv.Status.Version, createdSv.Status.Version)
+			},
+			"delete": func(t *rapid.T) {
+				ns := namespaceGen.Draw(t, "ns")
+				name := nameGen.Draw(t, "name")
+				modelSv, modelErr := model.delete(ns, name)
+				sv, err := sut.DeleteSv(t.Context(), ns, name)
+				if err != nil || modelErr != nil {
+					require.ErrorIs(t, err, modelErr)
+					return
+				}
+				require.Equal(t, modelSv.Namespace, sv.Namespace)
+				require.Equal(t, modelSv.Name, sv.Name)
+				require.Equal(t, modelSv.Status.Version, sv.Status.Version)
+			},
+			"lease": func(t *rapid.T) {
+				// Taken from secureValueMetadataStorage.acquireLeases
+				minAge := 300 * time.Second
+				leaseTTL := 30 * time.Second
+				maxBatchSize := rapid.Uint16Range(1, 10).Draw(t, "maxBatchSize")
+				modelSvs, modelErr := model.leaseInactiveSecureValues(sut.Clock.Now(), minAge, leaseTTL, maxBatchSize)
+				svs, err := sut.SecureValueMetadataStorage.LeaseInactiveSecureValues(t.Context(), maxBatchSize)
+				require.ErrorIs(t, err, modelErr)
+				require.Equal(t, len(modelSvs), len(svs))
+			},
+			"advanceTime": func(t *rapid.T) {
+				duration := time.Duration(rapid.IntRange(1, 10).Draw(t, "minutes")) * time.Minute
+				sut.Clock.AdvanceBy(duration)
+			},
+		})
+	})
+}

pkg/storage/secret/metadata/secure_value_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"slices"
 	"testing"
+	"time"
 
 	"github.com/mitchellh/copystructure"
 	"github.com/stretchr/testify/require"
@@ -20,7 +21,9 @@ import (
 
 type modelSecureValue struct {
 	*secretv1beta1.SecureValue
-	active bool
+	active       bool
+	created      time.Time
+	leaseCreated time.Time
 }
 
 // A simplified model of the grafana secrets manager
@@ -69,8 +72,8 @@ func (m *model) readActiveVersion(namespace, name string) *modelSecureValue {
 	return nil
 }
 
-func (m *model) create(sv *secretv1beta1.SecureValue, actorUID string) (*secretv1beta1.SecureValue, error) {
-	modelSv := &modelSecureValue{sv, false}
+func (m *model) create(now time.Time, sv *secretv1beta1.SecureValue) (*secretv1beta1.SecureValue, error) {
+	modelSv := &modelSecureValue{SecureValue: sv, active: false, created: now}
 	modelSv.Status.Version = m.getNewVersionNumber(modelSv.Namespace, modelSv.Name)
 	modelSv.Status.ExternalID = fmt.Sprintf("%d", modelSv.Status.Version)
 	m.secureValues = append(m.secureValues, modelSv)
@@ -78,7 +81,7 @@ func (m *model) create(sv *secretv1beta1.SecureValue, actorUID string) (*secretv
 	return modelSv.SecureValue, nil
 }
 
-func (m *model) update(newSecureValue *secretv1beta1.SecureValue, actorUID string) (*secretv1beta1.SecureValue, bool, error) {
+func (m *model) update(now time.Time, newSecureValue *secretv1beta1.SecureValue) (*secretv1beta1.SecureValue, bool, error) {
 	// If the payload doesn't contain a value, get the value from current version
 	if newSecureValue.Spec.Value == nil {
 		sv := m.readActiveVersion(newSecureValue.Namespace, newSecureValue.Name)
@@ -87,7 +90,7 @@ func (m *model) update(newSecureValue *secretv1beta1.SecureValue, actorUID strin
 		}
 		newSecureValue.Spec.Value = sv.Spec.Value
 	}
-	createdSv, err := m.create(newSecureValue, actorUID)
+	createdSv, err := m.create(now, newSecureValue)
 	return createdSv, true, err
 }
 
@@ -141,6 +144,22 @@ func (m *model) read(namespace, name string) (*secretv1beta1.SecureValue, error)
 	return modelSv.SecureValue, nil
 }
 
+func (m *model) leaseInactiveSecureValues(now time.Time, minAge, leaseTTL time.Duration, maxBatchSize uint16) ([]*modelSecureValue, error) {
+	out := make([]*modelSecureValue, 0)
+
+	for _, sv := range m.secureValues {
+		if len(out) >= int(maxBatchSize) {
+			break
+		}
+		if !sv.active && now.Sub(sv.created) > minAge && now.Sub(sv.leaseCreated) > leaseTTL {
+			sv.leaseCreated = now
+			out = append(out, sv)
+		}
+	}
+
+	return out, nil
+}
+
 var (
 	decryptersGen     = rapid.SampledFrom([]string{"svc1", "svc2", "svc3", "svc4", "svc5"})
 	nameGen           = rapid.SampledFrom([]string{"n1", "n2", "n3", "n4", "n5"})
@@ -204,16 +223,17 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
 		// Create a secure value
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 		require.Equal(t, sv.Namespace, sv1.Namespace)
 		require.Equal(t, sv.Name, sv1.Name)
 		require.EqualValues(t, 1, sv1.Status.Version)
 
 		// Create a new version of a secure value
-		sv2, err := m.create(deepCopy(sv), "actor-uid")
+		sv2, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 		require.Equal(t, sv.Namespace, sv2.Namespace)
 		require.Equal(t, sv.Name, sv2.Name)
@@ -225,11 +245,13 @@ func TestModel(t *testing.T) {
 
 		m := newModel()
 
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		now := time.Now()
+
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// Create a new version of a secure value by updating it
-		sv2, _, err := m.update(deepCopy(sv1), "actor-uid")
+		sv2, _, err := m.update(now, deepCopy(sv1))
 		require.NoError(t, err)
 		require.Equal(t, sv.Namespace, sv2.Namespace)
 		require.Equal(t, sv.Name, sv2.Name)
@@ -239,14 +261,14 @@ func TestModel(t *testing.T) {
 		sv3 := deepCopy(sv2)
 		sv3.Name = "i_dont_exist"
 		sv3.Spec.Value = nil
-		_, _, err = m.update(sv3, "actor-uid")
+		_, _, err = m.update(now, sv3)
 		require.ErrorIs(t, err, contracts.ErrSecureValueNotFound)
 
 		// Updating a value that doesn't exist creates a new version
 		sv4 := deepCopy(sv3)
 		sv4.Name = "i_dont_exist"
 		sv4.Spec.Value = ptr.To(secretv1beta1.NewExposedSecureValue("sv4"))
-		sv4, _, err = m.update(sv4, "actor-uid")
+		sv4, _, err = m.update(now, sv4)
 		require.NoError(t, err)
 		require.EqualValues(t, 1, sv4.Status.Version)
 	})
@@ -255,8 +277,9 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// Deleting a secure value
@@ -275,6 +298,7 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
 		// No secure values exist yet
 		list, err := m.list(sv.Namespace)
@@ -282,7 +306,7 @@ func TestModel(t *testing.T) {
 		require.Equal(t, 0, len(list.Items))
 
 		// Create a secure value
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// 1 secure value exists and it should be returned
@@ -298,6 +322,7 @@ func TestModel(t *testing.T) {
 		t.Parallel()
 
 		m := newModel()
+		now := time.Now()
 
 		// Decrypting a secure value that does not exist
 		result, err := m.decrypt("decrypter", "namespace", "name")
@@ -308,7 +333,7 @@ func TestModel(t *testing.T) {
 
 		// Create a secure value
 		secret := "v1"
-		sv1, err := m.create(deepCopy(sv), "actor-uid")
+		sv1, err := m.create(now, deepCopy(sv))
 		require.NoError(t, err)
 
 		// Decrypt the just created secure value
@@ -333,7 +358,7 @@ func TestStateMachine(t *testing.T) {
 			"create": func(t *rapid.T) {
 				sv := anySecureValueGen.Draw(t, "sv")
 
-				modelCreatedSv, modelErr := model.create(deepCopy(sv), "actor-uid")
+				modelCreatedSv, modelErr := model.create(sut.Clock.Now(), deepCopy(sv))
 
 				createdSv, err := sut.CreateSv(t.Context(), testutils.CreateSvWithSv(deepCopy(sv)))
 				if err != nil || modelErr != nil {
@@ -346,7 +371,7 @@ func TestStateMachine(t *testing.T) {
 			},
 			"update": func(t *rapid.T) {
 				sv := updateSecureValueGen.Draw(t, "sv")
-				modelCreatedSv, _, modelErr := model.update(deepCopy(sv), "actor-uid")
+				modelCreatedSv, _, modelErr := model.update(sut.Clock.Now(), deepCopy(sv))
 				createdSv, err := sut.UpdateSv(t.Context(), deepCopy(sv))
 				if err != nil || modelErr != nil {
 					require.ErrorIs(t, err, modelErr)

pkg/storage/secret/metadata/testdata/mysql--secure_value_delete-deleteSecureValue.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  `secret_secure_value`
+WHERE 
+  `namespace` = 'ns' AND
+  `name` = 'name' AND
+  `version` = 1
+;

pkg/storage/secret/metadata/testdata/mysql--secure_value_lease_inactive-lease inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  `secret_secure_value`
+SET
+  `lease_token` = 'token',
+  `lease_created` = 10
+WHERE 
+  `guid` IN (SELECT `guid` 
+                          FROM `secret_secure_value` 
+                          WHERE 
+                            `active` = FALSE AND
+                            10 - `created` > 300 AND
+                            10 - `lease_created` > 30
+                          LIMIT 10)
+;

pkg/storage/secret/metadata/testdata/mysql--secure_value_list_by_lease_token-list by lease token.sql

@@ -0,0 +1,26 @@
+SELECT
+  `guid`,
+  `name`,
+  `namespace`,
+  `annotations`,
+  `labels`,
+  `created`,
+  `created_by`,
+  `updated`,
+  `updated_by`,
+  `description`,
+  `keeper`,
+  `decrypters`,
+  `ref`,
+  `external_id`,
+  `version`,
+  `active`,
+  `owner_reference_api_group`,
+  `owner_reference_api_version`,
+  `owner_reference_kind`,
+  `owner_reference_name`
+FROM
+  `secret_secure_value`
+WHERE 
+  `lease_token` = 'token'
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_delete-deleteSecureValue.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  "secret_secure_value"
+WHERE 
+  "namespace" = 'ns' AND
+  "name" = 'name' AND
+  "version" = 1
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_lease_inactive-lease inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  "secret_secure_value"
+SET
+  "lease_token" = 'token',
+  "lease_created" = 10
+WHERE 
+  "guid" IN (SELECT "guid" 
+                          FROM "secret_secure_value" 
+                          WHERE 
+                            "active" = FALSE AND
+                            10 - "created" > 300 AND
+                            10 - "lease_created" > 30
+                          LIMIT 10)
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_list_by_lease_token-list by lease token.sql

@@ -0,0 +1,26 @@
+SELECT
+  "guid",
+  "name",
+  "namespace",
+  "annotations",
+  "labels",
+  "created",
+  "created_by",
+  "updated",
+  "updated_by",
+  "description",
+  "keeper",
+  "decrypters",
+  "ref",
+  "external_id",
+  "version",
+  "active",
+  "owner_reference_api_group",
+  "owner_reference_api_version",
+  "owner_reference_kind",
+  "owner_reference_name"
+FROM
+  "secret_secure_value"
+WHERE 
+  "lease_token" = 'token'
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_delete-deleteSecureValue.sql

@@ -0,0 +1,7 @@
+DELETE FROM 
+  "secret_secure_value"
+WHERE 
+  "namespace" = 'ns' AND
+  "name" = 'name' AND
+  "version" = 1
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_lease_inactive-lease inactive.sql

@@ -0,0 +1,14 @@
+UPDATE
+  "secret_secure_value"
+SET
+  "lease_token" = 'token',
+  "lease_created" = 10
+WHERE 
+  "guid" IN (SELECT "guid" 
+                          FROM "secret_secure_value" 
+                          WHERE 
+                            "active" = FALSE AND
+                            10 - "created" > 300 AND
+                            10 - "lease_created" > 30
+                          LIMIT 10)
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_list_by_lease_token-list by lease token.sql

@@ -0,0 +1,26 @@
+SELECT
+  "guid",
+  "name",
+  "namespace",
+  "annotations",
+  "labels",
+  "created",
+  "created_by",
+  "updated",
+  "updated_by",
+  "description",
+  "keeper",
+  "decrypters",
+  "ref",
+  "external_id",
+  "version",
+  "active",
+  "owner_reference_api_group",
+  "owner_reference_api_version",
+  "owner_reference_kind",
+  "owner_reference_name"
+FROM
+  "secret_secure_value"
+WHERE 
+  "lease_token" = 'token'
+;

pkg/storage/secret/migrator/migrator.go

@@ -181,4 +181,23 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 		Length:   253, // Limit enforced by K8s.
 		Nullable: true,
 	}))
+
+	mg.AddMigration("add lease_token column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "lease_token",
+		Type:     migrator.DB_NVarchar,
+		Length:   36,
+		Nullable: true,
+	}))
+	mg.AddMigration("add lease_token index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"lease_token"},
+	}))
+	mg.AddMigration("add lease_created column to "+TableNameSecureValue, migrator.NewAddColumnMigration(secureValueTable, &migrator.Column{
+		Name:     "lease_created",
+		Type:     migrator.DB_BigInt,
+		Nullable: false,
+		Default:  "0",
+	}))
+	mg.AddMigration("add lease_created index to "+TableNameSecureValue, migrator.NewAddIndexMigration(secureValueTable, &migrator.Index{
+		Cols: []string{"lease_created"},
+	}))
 }