Unified Storage: Add ListSinceModified to StorageBackend (#109697)

* WIP added ListSinceModified to StorageBackend interface * fix compile time check * Fix method name * Fix naming * fix the rest of the ListSinceModified names * Uses resource key without name field * get latest rv from resource_version. Update test. * adds moar tests * adds method stub for ListModifiedSince to other StorageBackend implementations * adds dummy impl to noop storage backend for ListModifiedSince * skip tests for badger kv backend for now * fixes tests and adds badgerkv impl for ListModifiedSince * add badger kv impl * adds test for new query * adds test data for new query * adds ListModifiedSince stub to mockStorageBackend * uncomment tests * refactors ListModifiedSince to return an iter.seq2 and handles deduplication. Updates tests. Updates query result sorting. * remove comments * remove folder from query (dont need it, yet?) * regen test queries * updates test * updates function comment * use resourcepb.ResourceKey instead of ModifiedResourceKey * wrap seq in single transaction. Rollback transaction after 30s if iterator never used. Only track last seen event. Formatting. * skip TestListModifiedSince for kv backend * use WatchEvent_Type for action type * remove redundant fields from order by clause and regen test data for query * remove redundant fields from order by clause and regen test data for query
2025-08-20 11:54:31 -06:00
parent 49739618fa
commit cace999671
15 changed files with 329 additions and 19 deletions
@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"iter"
 	"math"
 	"sync"
 	"time"
@@ -631,6 +632,99 @@ func (b *backend) listLatest(ctx context.Context, req *resourcepb.ListRequest, c
 	return iter.listRV, err
 }

+// ListModifiedSince will return all resources that have changed since the given resource version.
+// If a resource has changes, only the latest change will be returned.
+func (b *backend) ListModifiedSince(ctx context.Context, key resource.NamespacedResource, sinceRv int64) (int64, iter.Seq2[*resource.ModifiedResource, error]) {
+	tx, err := b.db.BeginTx(ctx, RepeatableRead)
+	if err != nil {
+		return 0, func(yield func(*resource.ModifiedResource, error) bool) {
+			yield(nil, err)
+		}
+	}
+
+	// Fetch latest RV within the transaction
+	latestRv, err := b.fetchLatestRV(ctx, tx, b.dialect, key.Group, key.Resource)
+	if err != nil {
+		terr := tx.Rollback()
+		if terr != nil {
+			b.log.Warn("Error rolling back transaction in ListModifiedSince", "error", terr)
+		}
+		return 0, func(yield func(*resource.ModifiedResource, error) bool) {
+			yield(nil, err)
+		}
+	}
+
+	// since results are sorted by name ASC and rv DESC, we can get away with tracking the last seen
+	lastSeen := ""
+
+	// rollback transaction if iterator not called within 30 seconds
+	rollbackTimer := time.AfterFunc(30*time.Second, func() {
+		if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
+			b.log.Warn("rollback timer error", "err", err)
+		}
+	})
+
+	seq := func(yield func(*resource.ModifiedResource, error) bool) {
+		rollbackTimer.Stop()
+
+		defer func() {
+			// Always rollback the read-only transaction when iterator is done
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				b.log.Warn("Error rolling back transaction in ListModifiedSince", "error", rollbackErr)
+			}
+		}()
+
+		query := sqlResourceListModifiedSinceRequest{
+			SQLTemplate: sqltemplate.New(b.dialect),
+			Namespace:   key.Namespace,
+			Group:       key.Group,
+			Resource:    key.Resource,
+			SinceRv:     sinceRv,
+		}
+
+		rows, err := dbutil.QueryRows(ctx, tx, sqlResourceHistoryListModifiedSince, query)
+		if err != nil {
+			yield(nil, err)
+			return
+		}
+		if rows != nil {
+			defer func() {
+				if cerr := rows.Close(); cerr != nil {
+					b.log.Warn("listSinceModified error closing rows", "error", cerr)
+				}
+			}()
+		}
+
+		for rows.Next() {
+			mr := &resource.ModifiedResource{}
+			if err := rows.Scan(&mr.Key.Namespace, &mr.Key.Group, &mr.Key.Resource, &mr.Key.Name, &mr.ResourceVersion, &mr.Action, &mr.Value); err != nil {
+				if !yield(nil, err) {
+					return
+				}
+				continue
+			}
+
+			// Deduplicate by name (namespace, group, and resource are always the same in the result set)
+			if mr.Key.Name == lastSeen {
+				continue
+			}
+
+			if mr.Key.Name <= lastSeen {
+				// resource names should be sorted alphabetically. So if not, the query is not correct.
+				yield(nil, fmt.Errorf("listModifiedSince: resources are not sorted by name ASC, lastSeen: %q, current: %q", lastSeen, mr.Key.Name))
+			}
+
+			lastSeen = mr.Key.Name
+
+			if !yield(mr, nil) {
+				return
+			}
+		}
+	}
+
+	return latestRv, seq
+}
+
 // listAtRevision fetches the resources from the resource_history table at a specific revision.
 func (b *backend) listAtRevision(ctx context.Context, req *resourcepb.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
 	ctx, span := b.tracer.Start(ctx, tracePrefix+"listAtRevision")