public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kvstore: update storage backend to use batch get (#112704)

Browse Source 
* consume batchget

* remove rv

* remove un-necessary limit

* do not pass stop to the iterators

* ensure we return the error
This commit is contained in:
Georges Chaudy
2025-10-21 15:42:07 +00:00
committed by GitHub
parent 0a2564c4d5
commit eca379e98e
3 changed files with 284 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/resource/storage_backend.go
+80 -78
View File
@@ -364,14 +364,14 @@ func (k *kvStorageBackend) ListIterator(ctx context.Context, req *resourcepb.Lis
break
}
}
// Create pull-style iterator from BatchGet
next, stop := iter.Pull2(k.dataStore.BatchGet(ctx, keys))
defer stop()
iter := kvListIterator{
keys: keys,
currentIndex: -1,
ctx: ctx,
listRV: listRV,
offset: offset,
limit: req.Limit + 1, // TODO: for now we need at least one more item. Fix the caller
dataStore: k.dataStore,
listRV: listRV,
offset: offset,
next: next,
}
err := cb(&iter)
if err != nil {
@@ -383,52 +383,44 @@ func (k *kvStorageBackend) ListIterator(ctx context.Context, req *resourcepb.Lis
// kvListIterator implements ListIterator for KV storage
type kvListIterator struct {
ctx context.Context
keys []DataKey
currentIndex int
dataStore *dataStore
listRV int64
offset int64
limit int64
listRV int64
offset int64
// current
rv int64
err error
value []byte
// pull-style iterator
next func() (DataObj, error, bool)
// current item state
currentDataObj *DataObj
value []byte
err error
}
func (i *kvListIterator) Next() bool {
i.currentIndex++
if i.currentIndex >= len(i.keys) {
// Pull next item from the iterator
dataObj, err, ok := i.next()
if !ok {
return false
}
if int64(i.currentIndex) >= i.limit {
return false
}
i.rv, i.err = i.keys[i.currentIndex].ResourceVersion, nil
data, err := i.dataStore.Get(i.ctx, i.keys[i.currentIndex])
if err != nil {
i.err = err
return false
}
i.value, i.err = readAndClose(data)
if i.err != nil {
i.currentDataObj = &dataObj
i.value, err = readAndClose(dataObj.Value)
if err != nil {
i.err = err
return false
}
// increment the offset
i.offset++
return true
}
func (i *kvListIterator) Error() error {
return nil
return i.err
}
func (i *kvListIterator) ContinueToken() string {
@@ -439,19 +431,31 @@ func (i *kvListIterator) ContinueToken() string {
}
func (i *kvListIterator) ResourceVersion() int64 {
return i.rv
if i.currentDataObj != nil {
return i.currentDataObj.Key.ResourceVersion
}
return 0
}
func (i *kvListIterator) Namespace() string {
return i.keys[i.currentIndex].Namespace
if i.currentDataObj != nil {
return i.currentDataObj.Key.Namespace
}
return ""
}
func (i *kvListIterator) Name() string {
return i.keys[i.currentIndex].Name
if i.currentDataObj != nil {
return i.currentDataObj.Key.Name
}
return ""
}
func (i *kvListIterator) Folder() string {
return i.keys[i.currentIndex].Folder
if i.currentDataObj != nil {
return i.currentDataObj.Key.Folder
}
return ""
}
func (i *kvListIterator) Value() []byte {
@@ -825,13 +829,14 @@ func (k *kvStorageBackend) ListHistory(ctx context.Context, req *resourcepb.List
// Pagination: filter out items up to and including lastSeenRV
pagedKeys := applyPagination(filteredKeys, lastSeenRV, sortAscending)
// Create pull-style iterator from BatchGet
next, stop := iter.Pull2(k.dataStore.BatchGet(ctx, pagedKeys))
defer stop()
iter := kvHistoryIterator{
keys: pagedKeys,
currentIndex: -1,
ctx: ctx,
listRV: listRV,
sortAscending: sortAscending,
dataStore: k.dataStore,
next: next,
}
err := fn(&iter)
@@ -889,14 +894,15 @@ func (k *kvStorageBackend) processTrashEntries(ctx context.Context, req *resourc
// Pagination: filter out items up to and including lastSeenRV
pagedKeys := applyPagination(filteredKeys, lastSeenRV, sortAscending)
// Create pull-style iterator from BatchGet
next, stop := iter.Pull2(k.dataStore.BatchGet(ctx, pagedKeys))
defer stop()
iter := kvHistoryIterator{
keys: pagedKeys,
currentIndex: -1,
ctx: ctx,
listRV: listRV,
sortAscending: sortAscending,
dataStore: k.dataStore,
skipProvisioned: true,
next: next,
}
err = fn(&iter)
@@ -909,43 +915,36 @@ func (k *kvStorageBackend) processTrashEntries(ctx context.Context, req *resourc
// kvHistoryIterator implements ListIterator for KV storage history
type kvHistoryIterator struct {
ctx context.Context
keys []DataKey
currentIndex int
listRV int64
sortAscending bool
skipProvisioned bool
dataStore *dataStore
// current
rv int64
err error
value []byte
folder string
// pull-style iterator
next func() (DataObj, error, bool)
// current item state
currentDataObj *DataObj
value []byte
folder string
err error
}
func (i *kvHistoryIterator) Next() bool {
i.currentIndex++
if i.currentIndex >= len(i.keys) {
// Pull next item from the iterator
dataObj, err, ok := i.next()
if !ok {
return false
}
key := i.keys[i.currentIndex]
i.rv = key.ResourceVersion
// Read the value from the ReadCloser
data, err := i.dataStore.Get(i.ctx, key)
if err != nil {
i.err = err
return false
}
if data == nil {
i.err = fmt.Errorf("data is nil")
return false
}
i.value, i.err = readAndClose(data)
if i.err != nil {
i.currentDataObj = &dataObj
i.value, err = readAndClose(dataObj.Value)
if err != nil {
i.err = err
return false
}
@@ -963,7 +962,6 @@ func (i *kvHistoryIterator) Next() bool {
return false
}
i.folder = meta.GetFolder()
i.err = nil
// if the resource is provisioned and we are skipping provisioned resources, continue onto the next one
if i.skipProvisioned && meta.GetAnnotation(utils.AnnoKeyManagerKind) != "" {
@@ -978,31 +976,35 @@ func (i *kvHistoryIterator) Error() error {
}
func (i *kvHistoryIterator) ContinueToken() string {
if i.currentIndex < 0 || i.currentIndex >= len(i.keys) {
if i.currentDataObj == nil {
return ""
}
rv := i.currentDataObj.Key.ResourceVersion
token := ContinueToken{
StartOffset: i.rv,
ResourceVersion: i.keys[i.currentIndex].ResourceVersion,
StartOffset: rv,
ResourceVersion: rv,
SortAscending: i.sortAscending,
}
return token.String()
}
func (i *kvHistoryIterator) ResourceVersion() int64 {
return i.rv
if i.currentDataObj != nil {
return i.currentDataObj.Key.ResourceVersion
}
return 0
}
func (i *kvHistoryIterator) Namespace() string {
if i.currentIndex >= 0 && i.currentIndex < len(i.keys) {
return i.keys[i.currentIndex].Namespace
if i.currentDataObj != nil {
return i.currentDataObj.Key.Namespace
}
return ""
}
func (i *kvHistoryIterator) Name() string {
if i.currentIndex >= 0 && i.currentIndex < len(i.keys) {
return i.keys[i.currentIndex].Name
if i.currentDataObj != nil {
return i.currentDataObj.Key.Name
}
return ""
}