public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unified Storage: Add ListSinceModified to StorageBackend (#109697)

Browse Source 
* WIP added ListSinceModified to StorageBackend interface

* fix compile time check

* Fix method name

* Fix naming

* fix the rest of the ListSinceModified names

* Uses resource key without name field

* get latest rv from resource_version. Update test.

* adds moar tests

* adds method stub for ListModifiedSince to other StorageBackend implementations

* adds dummy impl to noop storage backend for ListModifiedSince

* skip tests for badger kv backend for now

* fixes tests and adds badgerkv impl for ListModifiedSince

* add badger kv impl

* adds test for new query

* adds test data for new query

* adds ListModifiedSince stub to mockStorageBackend

* uncomment tests

* refactors ListModifiedSince to return an iter.seq2 and handles deduplication. Updates tests. Updates query result sorting.

* remove comments

* remove folder from query (dont need it, yet?)

* regen test queries

* updates test

* updates function comment

* use resourcepb.ResourceKey instead of ModifiedResourceKey

* wrap seq in single transaction. Rollback transaction after 30s if iterator never used. Only track last seen event. Formatting.

* skip TestListModifiedSince for kv backend

* use WatchEvent_Type for action type

* remove redundant fields from order by clause and regen test data for query

* remove redundant fields from order by clause and regen test data for query
This commit is contained in:
owensmallwood
2025-08-20 11:54:31 -06:00
committed by GitHub
parent 49739618fa
commit cace999671
15 changed files with 329 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/sql/backend.go
+94
View File
@@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"iter"
"math"
"sync"
"time"
@@ -631,6 +632,99 @@ func (b *backend) listLatest(ctx context.Context, req *resourcepb.ListRequest, c
return iter.listRV, err
}
// ListModifiedSince will return all resources that have changed since the given resource version.
// If a resource has changes, only the latest change will be returned.
func (b *backend) ListModifiedSince(ctx context.Context, key resource.NamespacedResource, sinceRv int64) (int64, iter.Seq2[*resource.ModifiedResource, error]) {
tx, err := b.db.BeginTx(ctx, RepeatableRead)
if err != nil {
return 0, func(yield func(*resource.ModifiedResource, error) bool) {
yield(nil, err)
}
}
// Fetch latest RV within the transaction
latestRv, err := b.fetchLatestRV(ctx, tx, b.dialect, key.Group, key.Resource)
if err != nil {
terr := tx.Rollback()
if terr != nil {
b.log.Warn("Error rolling back transaction in ListModifiedSince", "error", terr)
}
return 0, func(yield func(*resource.ModifiedResource, error) bool) {
yield(nil, err)
}
}
// since results are sorted by name ASC and rv DESC, we can get away with tracking the last seen
lastSeen := ""
// rollback transaction if iterator not called within 30 seconds
rollbackTimer := time.AfterFunc(30*time.Second, func() {
if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
b.log.Warn("rollback timer error", "err", err)
}
})
seq := func(yield func(*resource.ModifiedResource, error) bool) {
rollbackTimer.Stop()
defer func() {
// Always rollback the read-only transaction when iterator is done
if rollbackErr := tx.Rollback(); rollbackErr != nil {
b.log.Warn("Error rolling back transaction in ListModifiedSince", "error", rollbackErr)
}
}()
query := sqlResourceListModifiedSinceRequest{
SQLTemplate: sqltemplate.New(b.dialect),
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
SinceRv: sinceRv,
}
rows, err := dbutil.QueryRows(ctx, tx, sqlResourceHistoryListModifiedSince, query)
if err != nil {
yield(nil, err)
return
}
if rows != nil {
defer func() {
if cerr := rows.Close(); cerr != nil {
b.log.Warn("listSinceModified error closing rows", "error", cerr)
}
}()
}
for rows.Next() {
mr := &resource.ModifiedResource{}
if err := rows.Scan(&mr.Key.Namespace, &mr.Key.Group, &mr.Key.Resource, &mr.Key.Name, &mr.ResourceVersion, &mr.Action, &mr.Value); err != nil {
if !yield(nil, err) {
return
}
continue
}
// Deduplicate by name (namespace, group, and resource are always the same in the result set)
if mr.Key.Name == lastSeen {
continue
}
if mr.Key.Name <= lastSeen {
// resource names should be sorted alphabetically. So if not, the query is not correct.
yield(nil, fmt.Errorf("listModifiedSince: resources are not sorted by name ASC, lastSeen: %q, current: %q", lastSeen, mr.Key.Name))
}
lastSeen = mr.Key.Name
if !yield(mr, nil) {
return
}
}
}
return latestRv, seq
}
// listAtRevision fetches the resources from the resource_history table at a specific revision.
func (b *backend) listAtRevision(ctx context.Context, req *resourcepb.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
ctx, span := b.tracer.Start(ctx, tracePrefix+"listAtRevision")
pkg/storage/unified/sql/data/resource_history_list_since_modified.sql
+14
View File
@@ -0,0 +1,14 @@
SELECT
{{.Ident "namespace"}},
{{.Ident "group"}},
{{.Ident "resource"}},
{{.Ident "name"}},
{{.Ident "resource_version"}},
{{.Ident "action"}},
{{.Ident "value"}}
FROM resource_history
WHERE {{.Ident "namespace" }} = {{.Arg .Namespace }}
AND {{.Ident "group" }} = {{.Arg .Group }}
AND {{.Ident "resource" }} = {{.Arg .Resource }}
AND {{.Ident "resource_version" }} > {{.Arg .SinceRv }} -- needs to be exclusive of the sinceRv
ORDER BY {{.Ident "name" }} ASC, {{.Ident "resource_version" }} DESC
pkg/storage/unified/sql/queries.go
+43 -18
View File
@@ -30,24 +30,25 @@ func mustTemplate(filename string) *template.Template {
// Templates.
var (
sqlResourceDelete = mustTemplate("resource_delete.sql")
sqlResourceInsert = mustTemplate("resource_insert.sql")
sqlResourceUpdate = mustTemplate("resource_update.sql")
sqlResourceRead = mustTemplate("resource_read.sql")
sqlResourceStats = mustTemplate("resource_stats.sql")
sqlResourceList = mustTemplate("resource_list.sql")
sqlResourceHistoryList = mustTemplate("resource_history_list.sql")
sqlResourceUpdateRV = mustTemplate("resource_update_rv.sql")
sqlResourceHistoryRead = mustTemplate("resource_history_read.sql")
sqlResourceHistoryReadLatestRV = mustTemplate("resource_history_read_latest_rv.sql")
sqlResourceHistoryUpdateRV = mustTemplate("resource_history_update_rv.sql")
sqlResourceHistoryInsert = mustTemplate("resource_history_insert.sql")
sqlResourceHistoryPoll = mustTemplate("resource_history_poll.sql")
sqlResourceHistoryGet = mustTemplate("resource_history_get.sql")
sqlResourceHistoryDelete = mustTemplate("resource_history_delete.sql")
sqlResourceHistoryPrune = mustTemplate("resource_history_prune.sql")
sqlResourceTrash = mustTemplate("resource_trash.sql")
sqlResourceInsertFromHistory = mustTemplate("resource_insert_from_history.sql")
sqlResourceDelete = mustTemplate("resource_delete.sql")
sqlResourceInsert = mustTemplate("resource_insert.sql")
sqlResourceUpdate = mustTemplate("resource_update.sql")
sqlResourceRead = mustTemplate("resource_read.sql")
sqlResourceStats = mustTemplate("resource_stats.sql")
sqlResourceList = mustTemplate("resource_list.sql")
sqlResourceHistoryList = mustTemplate("resource_history_list.sql")
sqlResourceHistoryListModifiedSince = mustTemplate("resource_history_list_since_modified.sql")
sqlResourceUpdateRV = mustTemplate("resource_update_rv.sql")
sqlResourceHistoryRead = mustTemplate("resource_history_read.sql")
sqlResourceHistoryReadLatestRV = mustTemplate("resource_history_read_latest_rv.sql")
sqlResourceHistoryUpdateRV = mustTemplate("resource_history_update_rv.sql")
sqlResourceHistoryInsert = mustTemplate("resource_history_insert.sql")
sqlResourceHistoryPoll = mustTemplate("resource_history_poll.sql")
sqlResourceHistoryGet = mustTemplate("resource_history_get.sql")
sqlResourceHistoryDelete = mustTemplate("resource_history_delete.sql")
sqlResourceHistoryPrune = mustTemplate("resource_history_prune.sql")
sqlResourceTrash = mustTemplate("resource_trash.sql")
sqlResourceInsertFromHistory = mustTemplate("resource_insert_from_history.sql")
// sqlResourceLabelsInsert = mustTemplate("resource_labels_insert.sql")
sqlResourceVersionGet = mustTemplate("resource_version_get.sql")
@@ -425,3 +426,27 @@ func (r *sqlResourceVersionListRequest) Results() (*groupResourceVersion, error)
x := *r.groupResourceVersion
return &x, nil
}
type sqlResourceListModifiedSinceRequest struct {
sqltemplate.SQLTemplate
Namespace string
Group string
Resource string
SinceRv int64
}
func (r sqlResourceListModifiedSinceRequest) Validate() error {
if r.Namespace == "" {
return fmt.Errorf("missing namespace")
}
if r.Group == "" {
return fmt.Errorf("missing group")
}
if r.Resource == "" {
return fmt.Errorf("missing resource")
}
if r.SinceRv < 0 {
return fmt.Errorf("since resource version must be greater than or equal to zero")
}
return nil
}
pkg/storage/unified/sql/queries_test.go
+12
View File
@@ -120,6 +120,18 @@ func TestUnifiedStorageQueries(t *testing.T) {
},
},
},
sqlResourceHistoryListModifiedSince: {
{
Name: "single path",
Data: &sqlResourceListModifiedSinceRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Namespace: "ns",
Group: "group",
Resource: "res",
SinceRv: 10000,
},
},
},
sqlResourceHistoryPoll: {
{
Name: "single path",
pkg/storage/unified/sql/testdata/mysql--resource_history_list_since_modified-single path.sql
Vendored Executable
+14
View File
@@ -0,0 +1,14 @@
SELECT
`namespace`,
`group`,
`resource`,
`name`,
`resource_version`,
`action`,
`value`
FROM resource_history
WHERE `namespace` = 'ns'
AND `group` = 'group'
AND `resource` = 'res'
AND `resource_version` > 10000 -- needs to be exclusive of the sinceRv
ORDER BY `name` ASC, `resource_version` DESC
pkg/storage/unified/sql/testdata/postgres--resource_history_list_since_modified-single path.sql
Vendored Executable
+14
View File
@@ -0,0 +1,14 @@
SELECT
"namespace",
"group",
"resource",
"name",
"resource_version",
"action",
"value"
FROM resource_history
WHERE "namespace" = 'ns'
AND "group" = 'group'
AND "resource" = 'res'
AND "resource_version" > 10000 -- needs to be exclusive of the sinceRv
ORDER BY "name" ASC, "resource_version" DESC
pkg/storage/unified/sql/testdata/sqlite--resource_history_list_since_modified-single path.sql
Vendored Executable
+14
View File
@@ -0,0 +1,14 @@
SELECT
"namespace",
"group",
"resource",
"name",
"resource_version",
"action",
"value"
FROM resource_history
WHERE "namespace" = 'ns'
AND "group" = 'group'
AND "resource" = 'res'
AND "resource_version" > 10000 -- needs to be exclusive of the sinceRv
ORDER BY "name" ASC, "resource_version" DESC