public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unified Storage: Adds pruner to kv backend (#110549)

Browse Source 
* WIP adding pruner to kv store impl

* pruner only keeps 20 most recent versions

* ignore grafana-kv-data folder

* extracts some stuff to pruner.go file. Adds tests. Adds kvBackendOptions.

* update logging, comments, exports kvbackendoptions fields

* update nooppruner ref

* fixes field casing in test

* fix test

* linter fixes

* remove comment

* make KvStorageBackend private

* Adds pruner key validation and tests. Fixes broken tests.

* update error message when validating pruner key
This commit is contained in:
owensmallwood
2025-09-05 10:02:11 -06:00
committed by GitHub
parent f0095d84e3
commit d715bda8af
7 changed files with 386 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/sql/backend.go
+32 -55
View File
@@ -91,29 +91,6 @@ func NewBackend(opts BackendOptions) (Backend, error) {
}, nil
}
// pruningKey is a comparable key for pruning history.
type pruningKey struct {
namespace string
group string
resource string
name string
}
// Small abstraction to allow for different pruner implementations.
// This can be removed once the debouncer is deployed.
type pruner interface {
Add(key pruningKey) error
Start(ctx context.Context)
}
type noopPruner struct{}
func (p *noopPruner) Add(key pruningKey) error {
return nil
}
func (p *noopPruner) Start(ctx context.Context) {}
type backend struct {
//general
isHA bool
@@ -148,7 +125,7 @@ type backend struct {
// testing
simulatedNetworkLatency time.Duration
historyPruner pruner
historyPruner resource.Pruner
withPruner bool
}
@@ -205,26 +182,26 @@ func (b *backend) initLocked(ctx context.Context) error {
func (b *backend) initPruner(ctx context.Context) error {
if !b.withPruner {
b.log.Debug("using noop history pruner")
b.historyPruner = &noopPruner{}
b.historyPruner = &resource.NoopPruner{}
return nil
}
b.log.Debug("using debounced history pruner")
// Initialize history pruner.
pruner, err := debouncer.NewGroup(debouncer.DebouncerOpts[pruningKey]{
pruner, err := debouncer.NewGroup(debouncer.DebouncerOpts[resource.PruningKey]{
Name: "history_pruner",
BufferSize: 1000,
MinWait: time.Second * 30,
MaxWait: time.Minute * 5,
ProcessHandler: func(ctx context.Context, key pruningKey) error {
ProcessHandler: func(ctx context.Context, key resource.PruningKey) error {
return b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
res, err := dbutil.Exec(ctx, tx, sqlResourceHistoryPrune, &sqlPruneHistoryRequest{
SQLTemplate: sqltemplate.New(b.dialect),
HistoryLimit: defaultPrunerHistoryLimit,
Key: &resourcepb.ResourceKey{
Namespace: key.namespace,
Group: key.group,
Resource: key.resource,
Name: key.name,
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
Name: key.Name,
},
})
if err != nil {
@@ -235,20 +212,20 @@ func (b *backend) initPruner(ctx context.Context) error {
return fmt.Errorf("failed to get rows affected: %w", err)
}
b.log.Debug("pruned history successfully",
"namespace", key.namespace,
"group", key.group,
"resource", key.resource,
"name", key.name,
"namespace", key.Namespace,
"group", key.Group,
"resource", key.Resource,
"name", key.Name,
"rows", rows)
return nil
})
},
ErrorHandler: func(key pruningKey, err error) {
ErrorHandler: func(key resource.PruningKey, err error) {
b.log.Error("failed to prune history",
"namespace", key.namespace,
"group", key.group,
"resource", key.resource,
"name", key.name,
"namespace", key.Namespace,
"group", key.Group,
"resource", key.Resource,
"name", key.Name,
"error", err)
},
Reg: b.reg,
@@ -361,11 +338,11 @@ func (b *backend) create(ctx context.Context, event resource.WriteEvent) (int64,
}); err != nil {
return event.GUID, fmt.Errorf("insert into resource history: %w", err)
}
_ = b.historyPruner.Add(pruningKey{
namespace: event.Key.Namespace,
group: event.Key.Group,
resource: event.Key.Resource,
name: event.Key.Name,
_ = b.historyPruner.Add(resource.PruningKey{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
})
if b.simulatedNetworkLatency > 0 {
time.Sleep(b.simulatedNetworkLatency)
@@ -448,11 +425,11 @@ func (b *backend) update(ctx context.Context, event resource.WriteEvent) (int64,
}); err != nil {
return event.GUID, fmt.Errorf("insert into resource history: %w", err)
}
_ = b.historyPruner.Add(pruningKey{
namespace: event.Key.Namespace,
group: event.Key.Group,
resource: event.Key.Resource,
name: event.Key.Name,
_ = b.historyPruner.Add(resource.PruningKey{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
})
return event.GUID, nil
})
@@ -502,11 +479,11 @@ func (b *backend) delete(ctx context.Context, event resource.WriteEvent) (int64,
}); err != nil {
return event.GUID, fmt.Errorf("insert into resource history: %w", err)
}
_ = b.historyPruner.Add(pruningKey{
namespace: event.Key.Namespace,
group: event.Key.Group,
resource: event.Key.Resource,
name: event.Key.Name,
_ = b.historyPruner.Add(resource.PruningKey{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
})
return event.GUID, nil
})