Unified Storage: Add ListSinceModified to StorageBackend (#109697)

* WIP added ListSinceModified to StorageBackend interface

* fix compile time check

* Fix method name

* Fix naming

* fix the rest of the ListSinceModified names

* Uses resource key without name field

* get latest rv from resource_version. Update test.

* adds moar tests

* adds method stub for ListModifiedSince to other StorageBackend implementations

* adds dummy impl to noop storage backend for ListModifiedSince

* skip tests for badger kv backend for now

* fixes tests and adds badgerkv impl for ListModifiedSince

* add badger kv impl

* adds test for new query

* adds test data for new query

* adds ListModifiedSince stub to mockStorageBackend

* uncomment tests

* refactors ListModifiedSince to return an iter.seq2 and handles deduplication. Updates tests. Updates query result sorting.

* remove comments

* remove folder from query (dont need it, yet?)

* regen test queries

* updates test

* updates function comment

* use resourcepb.ResourceKey instead of ModifiedResourceKey

* wrap seq in single transaction. Rollback transaction after 30s if iterator never used. Only track last seen event. Formatting.

* skip TestListModifiedSince for kv backend

* use WatchEvent_Type for action type

* remove redundant fields from order by clause and regen test data for query

* remove redundant fields from order by clause and regen test data for query
This commit is contained in:
owensmallwood
2025-08-20 11:54:31 -06:00
committed by GitHub
parent 49739618fa
commit cace999671
15 changed files with 329 additions and 19 deletions
+12
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"iter"
"log/slog"
"net/http"
"sync"
@@ -114,6 +115,10 @@ type StorageBackend interface {
// ListHistory is like ListIterator, but it returns the history of a resource
ListHistory(context.Context, *resourcepb.ListRequest, func(ListIterator) error) (int64, error)
// ListModifiedSince will return all resources that have changed since the given resource version.
// If a resource has changes, only the latest change will be returned.
ListModifiedSince(ctx context.Context, key NamespacedResource, sinceRv int64) (int64, iter.Seq2[*ModifiedResource, error])
// Get all events from the store
// For HA setups, this will be more events than the local WriteEvent above!
WatchWriteEvents(ctx context.Context) (<-chan *WrittenEvent, error)
@@ -122,6 +127,13 @@ type StorageBackend interface {
GetResourceStats(ctx context.Context, namespace string, minCount int) ([]ResourceStats, error)
}
type ModifiedResource struct {
Action resourcepb.WatchEvent_Type
Key resourcepb.ResourceKey
Value []byte
ResourceVersion int64
}
type ResourceStats struct {
NamespacedResource