unistore: add kv based storage backend (#107305)

* Add datastore

* too many slashes

* lint

* add metadata store

* simplify meta

* Add eventstore

* golint

* lint

* Add datastore

* too many slashes

* lint

* pr comments

* extract ParseKey

* readcloser

* remove get prefix

* use dedicated keys

* parsekey

* sameresource

* unrelated

* name

* renmae tests

* add key validation

* fix tests

* refactor a bit

* lint

* allow empty ns

* get keys instead of list

* rename the functions

* refactor yield candidate

* update test

* unistore: add LastResourceVersion to datastore

* lint

* use map string

* missing err check

* fix

* Add storage backend

* remove hasmore

* fix tests

* small refactor

* pre-alloc

* extract the folder

* lint

* refactor

* handle context canceled in ListHistory to pass the tests

* fix the resource test

* unistore: provide generic tests for the kv interface (#107443)

unistore: move the kv tests to the testing package

* Update pkg/storage/unified/resource/storage_backend_test.go

Co-authored-by: Peter Štibraný <pstibrany@gmail.com>

* address comments

* comments

* comments

* comments

* normalise the names and add helper method

* events comments

* rename function

---------

Co-authored-by: Peter Štibraný <pstibrany@gmail.com>
This commit is contained in:
Georges Chaudy
2025-07-02 10:57:37 +00:00
committed by GitHub
co-authored by Peter Štibraný
parent f0a5829eb7
commit 696657bdd1
8 changed files with 2544 additions and 212 deletions
+490
View File
@@ -0,0 +1,490 @@
package test
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/util/testutil"
)
// Test names for the KV test suite
const (
TestKVGet = "get operations"
TestKVSave = "save operations"
TestKVDelete = "delete operations"
TestKVKeys = "keys listing"
TestKVKeysWithLimits = "keys with limits and ranges"
TestKVKeysWithSort = "keys with sorting"
TestKVConcurrent = "concurrent operations"
TestKVUnixTimestamp = "unix timestamp"
)
// NewKVFunc is a function that creates a new KV instance for testing
type NewKVFunc func(ctx context.Context) resource.KV
// KVTestOptions configures which tests to run
type KVTestOptions struct {
NSPrefix string // namespace prefix for isolation
}
// GenerateRandomKVPrefix creates a random namespace prefix for test isolation
func GenerateRandomKVPrefix() string {
return fmt.Sprintf("kvtest-%d", time.Now().UnixNano())
}
// RunKVTest runs the KV test suite
func RunKVTest(t *testing.T, newKV NewKVFunc, opts *KVTestOptions) {
if testing.Short() {
t.Skip("skipping integration test")
}
if opts == nil {
opts = &KVTestOptions{}
}
if opts.NSPrefix == "" {
opts.NSPrefix = GenerateRandomKVPrefix()
}
t.Logf("Running KV tests with namespace prefix: %s", opts.NSPrefix)
cases := []struct {
name string
fn func(*testing.T, resource.KV, string)
}{
{TestKVGet, runTestKVGet},
{TestKVSave, runTestKVSave},
{TestKVDelete, runTestKVDelete},
{TestKVKeys, runTestKVKeys},
{TestKVKeysWithLimits, runTestKVKeysWithLimits},
{TestKVKeysWithSort, runTestKVKeysWithSort},
{TestKVConcurrent, runTestKVConcurrent},
{TestKVUnixTimestamp, runTestKVUnixTimestamp},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
tc.fn(t, newKV(context.Background()), opts.NSPrefix)
})
}
}
func runTestKVGet(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-get"
t.Run("get existing key", func(t *testing.T) {
// First save a key
testValue := "test value for get"
err := kv.Save(ctx, section, "existing-key", strings.NewReader(testValue))
require.NoError(t, err)
// Now get it
obj, err := kv.Get(ctx, section, "existing-key")
require.NoError(t, err)
assert.Equal(t, "existing-key", obj.Key)
// Read the value
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, testValue, string(value))
// Close the value reader
err = obj.Value.Close()
require.NoError(t, err)
})
t.Run("get non-existent key", func(t *testing.T) {
_, err := kv.Get(ctx, section, "non-existent-key")
assert.Error(t, err)
assert.Equal(t, resource.ErrNotFound, err)
})
t.Run("get with empty section", func(t *testing.T) {
_, err := kv.Get(ctx, "", "some-key")
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
}
func runTestKVSave(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-save"
t.Run("save new key", func(t *testing.T) {
testValue := "new test value"
err := kv.Save(ctx, section, "new-key", strings.NewReader(testValue))
require.NoError(t, err)
// Verify it was saved
obj, err := kv.Get(ctx, section, "new-key")
require.NoError(t, err)
assert.Equal(t, "new-key", obj.Key)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, testValue, string(value))
err = obj.Value.Close()
require.NoError(t, err)
})
t.Run("save overwrite existing key", func(t *testing.T) {
// First save
err := kv.Save(ctx, section, "overwrite-key", strings.NewReader("old value"))
require.NoError(t, err)
// Overwrite
newValue := "new value"
err = kv.Save(ctx, section, "overwrite-key", strings.NewReader(newValue))
require.NoError(t, err)
// Verify it was updated
obj, err := kv.Get(ctx, section, "overwrite-key")
require.NoError(t, err)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, newValue, string(value))
err = obj.Value.Close()
require.NoError(t, err)
})
t.Run("save with empty section", func(t *testing.T) {
err := kv.Save(ctx, "", "some-key", strings.NewReader("some value"))
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
t.Run("save binary data", func(t *testing.T) {
binaryData := []byte{0x00, 0x01, 0x02, 0x03, 0xFF, 0xFE, 0xFD}
err := kv.Save(ctx, section, "binary-key", bytes.NewReader(binaryData))
require.NoError(t, err)
// Verify binary data
obj, err := kv.Get(ctx, section, "binary-key")
require.NoError(t, err)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, binaryData, value)
err = obj.Value.Close()
require.NoError(t, err)
})
}
func runTestKVDelete(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-delete"
t.Run("delete existing key", func(t *testing.T) {
// First create a key
err := kv.Save(ctx, section, "delete-key", strings.NewReader("delete me"))
require.NoError(t, err)
// Verify it exists
_, err = kv.Get(ctx, section, "delete-key")
require.NoError(t, err)
// Delete it
err = kv.Delete(ctx, section, "delete-key")
require.NoError(t, err)
// Verify it's gone
_, err = kv.Get(ctx, section, "delete-key")
assert.Error(t, err)
assert.Equal(t, resource.ErrNotFound, err)
})
t.Run("delete non-existent key", func(t *testing.T) {
err := kv.Delete(ctx, section, "non-existent-delete-key")
assert.Error(t, err)
assert.Equal(t, resource.ErrNotFound, err)
})
t.Run("delete with empty section", func(t *testing.T) {
err := kv.Delete(ctx, "", "some-key")
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
}
func runTestKVKeys(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-keys"
// Setup test data
testKeys := []string{"a1", "a2", "b1", "b2", "c1"}
for _, key := range testKeys {
err := kv.Save(ctx, section, key, strings.NewReader("value"+key))
require.NoError(t, err)
}
t.Run("list all keys", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, testKeys, keys)
})
t.Run("list keys with empty section", func(t *testing.T) {
var keys []string
var errors []error
for k, err := range kv.Keys(ctx, "", resource.ListOptions{}) {
if err != nil {
errors = append(errors, err)
break
}
keys = append(keys, k)
}
assert.Len(t, errors, 1)
assert.Contains(t, errors[0].Error(), "section is required")
assert.Empty(t, keys)
})
}
func runTestKVKeysWithLimits(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-keys-limits"
// Setup test data
testKeys := []string{"a1", "a2", "b1", "b2", "c1", "c2", "d1", "d2"}
for _, key := range testKeys {
err := kv.Save(ctx, section, key, strings.NewReader("value"+key))
require.NoError(t, err)
}
t.Run("keys with limit", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{Limit: 3}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a1", "a2", "b1"}, keys)
})
t.Run("keys with range", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{StartKey: "b", EndKey: "d"}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"b1", "b2", "c1", "c2"}, keys)
})
t.Run("keys with prefix", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
StartKey: "c",
EndKey: resource.PrefixRangeEnd("c"),
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"c1", "c2"}, keys)
})
t.Run("keys with limit and range", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
StartKey: "a",
EndKey: "c",
Limit: 2,
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a1", "a2"}, keys)
})
}
func runTestKVKeysWithSort(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-keys-sort"
// Setup test data
testKeys := []string{"a1", "a2", "b1", "b2", "c1"}
for _, key := range testKeys {
err := kv.Save(ctx, section, key, strings.NewReader("value"+key))
require.NoError(t, err)
}
t.Run("keys in ascending order (default)", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{Sort: resource.SortOrderAsc}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a1", "a2", "b1", "b2", "c1"}, keys)
})
t.Run("keys in descending order", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{Sort: resource.SortOrderDesc}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"c1", "b2", "b1", "a2", "a1"}, keys)
})
t.Run("keys descending with prefix", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
StartKey: "a",
EndKey: resource.PrefixRangeEnd("a"),
Sort: resource.SortOrderDesc,
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a2", "a1"}, keys)
})
t.Run("keys descending with limit", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
Sort: resource.SortOrderDesc,
Limit: 3,
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"c1", "b2", "b1"}, keys)
})
}
func runTestKVConcurrent(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(60*time.Second))
section := nsPrefix + "-concurrent"
t.Run("concurrent save and get operations", func(t *testing.T) {
const numGoroutines = 10
const numOperations = 20
done := make(chan error, numGoroutines)
for i := 0; i < numGoroutines; i++ {
go func(goroutineID int) {
var err error
defer func() { done <- err }()
for j := 0; j < numOperations; j++ {
key := fmt.Sprintf("concurrent-key-%d-%d", goroutineID, j)
value := fmt.Sprintf("concurrent-value-%d-%d", goroutineID, j)
// Save
err = kv.Save(ctx, section, key, strings.NewReader(value))
if err != nil {
return
}
// Get immediately
obj, err := kv.Get(ctx, section, key)
if err != nil {
return
}
readValue, err := io.ReadAll(obj.Value)
require.NoError(t, err)
err = obj.Value.Close()
require.NoError(t, err)
assert.Equal(t, value, string(readValue))
}
}(i)
}
// Wait for all goroutines to complete
for i := 0; i < numGoroutines; i++ {
err := <-done
require.NoError(t, err)
}
})
t.Run("concurrent save, delete, and list operations", func(t *testing.T) {
const numGoroutines = 5
done := make(chan error, numGoroutines)
for i := 0; i < numGoroutines; i++ {
go func(goroutineID int) {
var err error
defer func() { done <- err }()
key := fmt.Sprintf("concurrent-ops-key-%d", goroutineID)
value := fmt.Sprintf("concurrent-ops-value-%d", goroutineID)
// Save
err = kv.Save(ctx, section, key, strings.NewReader(value))
if err != nil {
return
}
// List to verify it exists
found := false
for k, err := range kv.Keys(ctx, section, resource.ListOptions{}) {
if err != nil {
return
}
if k == key {
found = true
break
}
}
if !found {
err = fmt.Errorf("key %s not found in list", key)
return
}
// Delete
err = kv.Delete(ctx, section, key)
if err != nil {
return
}
// Verify it's deleted
_, err = kv.Get(ctx, section, key)
require.ErrorIs(t, resource.ErrNotFound, err)
err = nil // Expected error, so clear it
}(i)
}
// Wait for all goroutines to complete
for i := 0; i < numGoroutines; i++ {
err := <-done
require.NoError(t, err)
}
})
}
func runTestKVUnixTimestamp(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
t.Run("unix timestamp returns reasonable value", func(t *testing.T) {
timestamp, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
now := time.Now().Unix()
// Allow for some time difference (up to 5 seconds)
assert.InDelta(t, now, timestamp, 5)
})
t.Run("unix timestamp is consistent", func(t *testing.T) {
timestamp1, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
timestamp2, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
// Should be very close (within 1 second)
require.InDelta(t, timestamp1, timestamp2, 1)
})
}
+28
View File
@@ -0,0 +1,28 @@
package test
import (
"context"
"testing"
badger "github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
func TestBadgerKV(t *testing.T) {
RunKVTest(t, func(ctx context.Context) resource.KV {
opts := badger.DefaultOptions("").WithInMemory(true).WithLogger(nil)
db, err := badger.Open(opts)
require.NoError(t, err)
t.Cleanup(func() {
err := db.Close()
require.NoError(t, err)
})
return resource.NewBadgerKV(db)
}, &KVTestOptions{
NSPrefix: "badger-kv-test",
})
}
@@ -55,10 +55,6 @@ func GenerateRandomNSPrefix() string {
// RunStorageBackendTest runs the storage backend test suite
func RunStorageBackendTest(t *testing.T, newBackend NewBackendFunc, opts *TestOptions) {
if testing.Short() {
t.Skip("skipping integration test")
}
if opts == nil {
opts = &TestOptions{}
}
@@ -987,10 +983,10 @@ func runTestIntegrationBackendCreateNewResource(t *testing.T, backend resource.S
Key: &resourcepb.ResourceKey{
Namespace: "default",
Group: "test.grafana",
Resource: "Test",
Resource: "tests",
Name: "test",
},
Value: []byte(`{"apiVersion":"test.grafana/v0alpha1","kind":"Test","metadata":{"name":"test","namespace":"default"}}`),
Value: []byte(`{"apiVersion":"test.grafana/v0alpha1","kind":"Test","metadata":{"name":"test","namespace":"default","uid":"test-uid-123"}}`),
}
response, err := server.Create(ctx, request)
@@ -0,0 +1,29 @@
package test
import (
"context"
"testing"
badger "github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
func TestBadgerKVStorageBackend(t *testing.T) {
RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend {
opts := badger.DefaultOptions("").WithInMemory(true).WithLogger(nil)
db, err := badger.Open(opts)
require.NoError(t, err)
t.Cleanup(func() {
_ = db.Close()
})
return resource.NewKvStorageBackend(resource.NewBadgerKV(db))
}, &TestOptions{
NSPrefix: "kvstorage-test",
SkipTests: map[string]bool{
// TODO: fix these tests and remove this skip
TestBlobSupport: true,
},
})
}