Unified Storage: Adds pruner to kv backend (#110549)

* WIP adding pruner to kv store impl

* pruner only keeps 20 most recent versions

* ignore grafana-kv-data folder

* extracts some stuff to pruner.go file. Adds tests. Adds kvBackendOptions.

* update logging, comments, exports kvbackendoptions fields

* update nooppruner ref

* fixes field casing in test

* fix test

* linter fixes

* remove comment

* make KvStorageBackend private

* Adds pruner key validation and tests. Fixes broken tests.

* update error message when validating pruner key

pkg/storage/unified/resource/storage_backend.go

@@ -18,34 +18,52 @@ import (
 	"github.com/grafana/grafana-app-sdk/logging"
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
+	"github.com/grafana/grafana/pkg/util/debouncer"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/trace"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
 	defaultListBufferSize = 100
+	prunerMaxEvents       = 20
 )
 
-// Unified storage backend based on KV storage.
+// kvStorageBackend Unified storage backend based on KV storage.
 type kvStorageBackend struct {
-	snowflake  *snowflake.Node
-	kv         KV
-	dataStore  *dataStore
-	metaStore  *metadataStore
-	eventStore *eventStore
-	notifier   *notifier
-	builder    DocumentBuilder
-	log        logging.Logger
+	snowflake     *snowflake.Node
+	kv            KV
+	dataStore     *dataStore
+	metaStore     *metadataStore
+	eventStore    *eventStore
+	notifier      *notifier
+	builder       DocumentBuilder
+	log           logging.Logger
+	withPruner    bool
+	historyPruner Pruner
+	//tracer        trace.Tracer
+	//reg           prometheus.Registerer
 }
 
 var _ StorageBackend = &kvStorageBackend{}
 
-func NewKvStorageBackend(kv KV) *kvStorageBackend {
+type KvBackendOptions struct {
+	KvStore    KV
+	WithPruner bool
+	Tracer     trace.Tracer          // TODO add tracing
+	Reg        prometheus.Registerer // TODO add metrics
+}
+
+func NewKvStorageBackend(opts KvBackendOptions) (StorageBackend, error) {
+	ctx := context.Background()
+	kv := opts.KvStore
+
 	s, err := snowflake.NewNode(rand.Int64N(1024))
 	if err != nil {
-		panic(err)
+		return nil, fmt.Errorf("failed to create snowflake node: %w", err)
 	}
 	eventStore := newEventStore(kv)
-	return &kvStorageBackend{
+	backend := &kvStorageBackend{
 		kv:         kv,
 		dataStore:  newDataStore(kv),
 		metaStore:  newMetadataStore(kv),
@@ -55,6 +73,72 @@ func NewKvStorageBackend(kv KV) *kvStorageBackend {
 		builder:    StandardDocumentBuilder(), // For now we use the standard document builder.
 		log:        &logging.NoOpLogger{},     // Make this configurable
 	}
+	err = backend.initPruner(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize pruner: %w", err)
+	}
+	return backend, nil
+}
+
+func (k *kvStorageBackend) pruneEvents(ctx context.Context, key PruningKey) error {
+	if !key.Validate() {
+		return fmt.Errorf("invalid pruning key, all fields must be set: %+v", key)
+	}
+
+	keepEvents := make([]DataKey, 0, prunerMaxEvents)
+
+	// iterate over all keys for the resource and delete versions beyond the latest 20
+	for datakey, err := range k.dataStore.Keys(ctx, ListRequestKey(key)) {
+		if err != nil {
+			return err
+		}
+
+		if len(keepEvents) < prunerMaxEvents {
+			keepEvents = append(keepEvents, datakey)
+			continue
+		}
+
+		// If we already have 20 versions, delete the oldest one and append the new one
+		err := k.dataStore.Delete(ctx, keepEvents[0])
+		if err != nil {
+			return err
+		}
+		keepEvents = append(keepEvents[1:], datakey)
+	}
+
+	return nil
+}
+
+func (k *kvStorageBackend) initPruner(ctx context.Context) error {
+	if !k.withPruner {
+		k.log.Debug("Pruner disabled, using noop pruner")
+		k.historyPruner = &NoopPruner{}
+		return nil
+	}
+
+	k.log.Debug("Initializing history pruner")
+	pruner, err := debouncer.NewGroup(debouncer.DebouncerOpts[PruningKey]{
+		Name:           "history_pruner",
+		BufferSize:     1000,
+		MinWait:        time.Second * 30,
+		MaxWait:        time.Minute * 5,
+		ProcessHandler: k.pruneEvents,
+		ErrorHandler: func(key PruningKey, err error) {
+			k.log.Error("failed to prune history",
+				"namespace", key.Namespace,
+				"group", key.Group,
+				"resource", key.Resource,
+				"name", key.Name,
+				"error", err)
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	k.historyPruner = pruner
+	k.historyPruner.Start(ctx)
+	return nil
 }
 
 // WriteEvent writes a resource event (create/update/delete) to the storage backend.
@@ -150,6 +234,14 @@ func (k *kvStorageBackend) WriteEvent(ctx context.Context, event WriteEvent) (in
 	if err != nil {
 		return 0, fmt.Errorf("failed to save event: %w", err)
 	}
+
+	_ = k.historyPruner.Add(PruningKey{
+		Namespace: event.Key.Namespace,
+		Group:     event.Key.Group,
+		Resource:  event.Key.Resource,
+		Name:      event.Key.Name,
+	})
+
 	return rv, nil
 }