Will Assis
755b479be4
* unified-storage: update resource_history_update_rv.sql to populate key_path in resource_history
1981 lines
66 KiB
Go
1981 lines
66 KiB
Go
package test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"iter"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/bwmarrin/snowflake"
|
|
"github.com/go-jose/go-jose/v4/jwt"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apiserver/pkg/endpoints/request"
|
|
|
|
"github.com/grafana/authlib/authn"
|
|
"github.com/grafana/authlib/types"
|
|
|
|
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
|
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
|
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
|
sqldb "github.com/grafana/grafana/pkg/storage/unified/sql/db"
|
|
"github.com/grafana/grafana/pkg/util/testutil"
|
|
)
|
|
|
|
// Test names for the storage backend test suite
|
|
const (
|
|
TestHappyPath = "happy path"
|
|
TestWatchWriteEvents = "watch write events from latest"
|
|
TestList = "list"
|
|
TestBlobSupport = "blob support"
|
|
TestGetResourceStats = "get resource stats"
|
|
TestListHistory = "list history"
|
|
TestListHistoryErrorReporting = "list history error reporting"
|
|
TestListModifiedSince = "list events since rv"
|
|
TestListTrash = "list trash"
|
|
TestCreateNewResource = "create new resource"
|
|
TestGetResourceLastImportTime = "get resource last import time"
|
|
TestOptimisticLocking = "optimistic locking on concurrent writes"
|
|
TestKeyPathGeneration = "key_path generation"
|
|
)
|
|
|
|
type NewBackendFunc func(ctx context.Context) resource.StorageBackend
|
|
|
|
// NewBackendWithDBFunc creates a backend with database access for testing
|
|
type NewBackendWithDBFunc func(ctx context.Context) (resource.StorageBackend, sqldb.DB)
|
|
|
|
// TestOptions configures which tests to run
|
|
type TestOptions struct {
|
|
SkipTests map[string]bool // tests to skip
|
|
NSPrefix string // namespace prefix for isolation
|
|
}
|
|
|
|
// GenerateRandomNSPrefix creates a random namespace prefix for test isolation
|
|
func GenerateRandomNSPrefix() string {
|
|
uid := uuid.New().String()[:10]
|
|
return fmt.Sprintf("test-%s", uid)
|
|
}
|
|
|
|
// RunStorageBackendTest runs the storage backend test suite
|
|
func RunStorageBackendTest(t *testing.T, newBackend NewBackendFunc, opts *TestOptions) {
|
|
if opts == nil {
|
|
opts = &TestOptions{}
|
|
}
|
|
|
|
if opts.NSPrefix == "" {
|
|
opts.NSPrefix = GenerateRandomNSPrefix()
|
|
}
|
|
|
|
t.Logf("Running tests with namespace prefix: %s", opts.NSPrefix)
|
|
|
|
cases := []struct {
|
|
name string
|
|
fn func(*testing.T, resource.StorageBackend, string)
|
|
}{
|
|
{TestHappyPath, runTestIntegrationBackendHappyPath},
|
|
{TestWatchWriteEvents, runTestIntegrationBackendWatchWriteEvents},
|
|
{TestList, runTestIntegrationBackendList},
|
|
{TestBlobSupport, runTestIntegrationBlobSupport},
|
|
{TestGetResourceStats, runTestIntegrationBackendGetResourceStats},
|
|
{TestListHistory, runTestIntegrationBackendListHistory},
|
|
{TestListHistoryErrorReporting, runTestIntegrationBackendListHistoryErrorReporting},
|
|
{TestListTrash, runTestIntegrationBackendTrash},
|
|
{TestCreateNewResource, runTestIntegrationBackendCreateNewResource},
|
|
{TestListModifiedSince, runTestIntegrationBackendListModifiedSince},
|
|
{TestGetResourceLastImportTime, runTestIntegrationGetResourceLastImportTime},
|
|
{TestOptimisticLocking, runTestIntegrationBackendOptimisticLocking},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
if shouldSkip := opts.SkipTests[tc.name]; shouldSkip {
|
|
t.Logf("Skipping test: %s", tc.name)
|
|
continue
|
|
}
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
tc.fn(t, newBackend(context.Background()), opts.NSPrefix)
|
|
})
|
|
}
|
|
}
|
|
|
|
func RunSQLStorageBackendCompatibilityTest(t *testing.T, newBackend NewBackendWithDBFunc, opts *TestOptions) {
|
|
if opts == nil {
|
|
opts = &TestOptions{}
|
|
}
|
|
|
|
if opts.NSPrefix == "" {
|
|
opts.NSPrefix = GenerateRandomNSPrefix()
|
|
}
|
|
|
|
t.Logf("Running tests with namespace prefix: %s", opts.NSPrefix)
|
|
|
|
cases := []struct {
|
|
name string
|
|
fn func(*testing.T, resource.StorageBackend, string, sqldb.DB)
|
|
}{
|
|
{TestKeyPathGeneration, runTestIntegrationBackendKeyPathGeneration},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
if shouldSkip := opts.SkipTests[tc.name]; shouldSkip {
|
|
t.Logf("Skipping test: %s", tc.name)
|
|
continue
|
|
}
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
backend, db := newBackend(context.Background())
|
|
tc.fn(t, backend, opts.NSPrefix, db)
|
|
})
|
|
}
|
|
}
|
|
|
|
func runTestIntegrationBackendHappyPath(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := types.WithAuthInfo(context.Background(), authn.NewAccessTokenAuthInfo(authn.Claims[authn.AccessTokenClaims]{
|
|
Claims: jwt.Claims{
|
|
Subject: "testuser",
|
|
},
|
|
Rest: authn.AccessTokenClaims{},
|
|
}))
|
|
|
|
server := newServer(t, backend)
|
|
ns := nsPrefix + "-ns1"
|
|
stream, err := backend.WatchWriteEvents(context.Background()) // Using a different context to avoid canceling the stream after the DefaultContextTimeout
|
|
require.NoError(t, err)
|
|
var rv1, rv2, rv3, rv4, rv5 int64
|
|
|
|
t.Run("Add 3 resources", func(t *testing.T) {
|
|
rv1, err = writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv1, int64(0))
|
|
|
|
rv2, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv2, rv1)
|
|
|
|
rv3, err = writeEvent(ctx, backend, "item3", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv3, rv2)
|
|
})
|
|
|
|
t.Run("Update item2", func(t *testing.T) {
|
|
rv4, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rv2))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv4, rv3)
|
|
})
|
|
|
|
t.Run("Delete item1", func(t *testing.T) {
|
|
rv5, err = writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rv1))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv5, rv4)
|
|
})
|
|
|
|
t.Run("Read latest item 2", func(t *testing.T) {
|
|
resp := backend.ReadResource(ctx, &resourcepb.ReadRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Name: "item2",
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
})
|
|
require.Nil(t, resp.Error)
|
|
require.Equal(t, rv4, resp.ResourceVersion)
|
|
require.Contains(t, string(resp.Value), "item2 MODIFIED")
|
|
require.Equal(t, "folderuid", resp.Folder)
|
|
})
|
|
|
|
t.Run("Read early version of item2", func(t *testing.T) {
|
|
resp := backend.ReadResource(ctx, &resourcepb.ReadRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Name: "item2",
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
ResourceVersion: rv3, // item2 was created at rv2 and updated at rv4
|
|
})
|
|
require.Nil(t, resp.Error)
|
|
require.Equal(t, rv2, resp.ResourceVersion)
|
|
require.Contains(t, string(resp.Value), "item2 ADDED")
|
|
})
|
|
|
|
t.Run("List latest", func(t *testing.T) {
|
|
resp, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, resp.Error)
|
|
require.Len(t, resp.Items, 2)
|
|
require.Contains(t, string(resp.Items[0].Value), "item2 MODIFIED")
|
|
require.Contains(t, string(resp.Items[1].Value), "item3 ADDED")
|
|
require.GreaterOrEqual(t, resp.ResourceVersion, rv5) // rv5 is the latest resource version
|
|
})
|
|
|
|
t.Run("Watch events", func(t *testing.T) {
|
|
event := <-stream
|
|
require.Equal(t, "item1", event.Key.Name)
|
|
require.Equal(t, rv1, event.ResourceVersion)
|
|
require.Equal(t, resourcepb.WatchEvent_ADDED, event.Type)
|
|
require.Equal(t, "folderuid", event.Folder)
|
|
|
|
event = <-stream
|
|
require.Equal(t, "item2", event.Key.Name)
|
|
require.Equal(t, rv2, event.ResourceVersion)
|
|
require.Equal(t, resourcepb.WatchEvent_ADDED, event.Type)
|
|
require.Equal(t, "folderuid", event.Folder)
|
|
|
|
event = <-stream
|
|
require.Equal(t, "item3", event.Key.Name)
|
|
require.Equal(t, rv3, event.ResourceVersion)
|
|
require.Equal(t, resourcepb.WatchEvent_ADDED, event.Type)
|
|
|
|
event = <-stream
|
|
require.Equal(t, "item2", event.Key.Name)
|
|
require.Equal(t, rv4, event.ResourceVersion)
|
|
require.Equal(t, resourcepb.WatchEvent_MODIFIED, event.Type)
|
|
|
|
event = <-stream
|
|
require.Equal(t, "item1", event.Key.Name)
|
|
require.Equal(t, rv5, event.ResourceVersion)
|
|
require.Equal(t, resourcepb.WatchEvent_DELETED, event.Type)
|
|
})
|
|
}
|
|
|
|
func runTestIntegrationBackendGetResourceStats(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
|
|
|
|
sortFunc := func(a, b resource.ResourceStats) int {
|
|
if a.Namespace != b.Namespace {
|
|
return strings.Compare(a.Namespace, b.Namespace)
|
|
}
|
|
if a.Group != b.Group {
|
|
return strings.Compare(a.Group, b.Group)
|
|
}
|
|
return strings.Compare(a.Resource, b.Resource)
|
|
}
|
|
// Create resources across different namespaces/groups
|
|
_, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED,
|
|
WithNamespace(nsPrefix+"-stats-ns1"),
|
|
WithGroup("group"),
|
|
WithResource("resource1"))
|
|
require.NoError(t, err)
|
|
|
|
_, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED,
|
|
WithNamespace(nsPrefix+"-stats-ns1"),
|
|
WithGroup("group"),
|
|
WithResource("resource1"))
|
|
require.NoError(t, err)
|
|
|
|
_, err = writeEvent(ctx, backend, "item3", resourcepb.WatchEvent_ADDED,
|
|
WithNamespace(nsPrefix+"-stats-ns1"),
|
|
WithGroup("group"),
|
|
WithResource("resource2"))
|
|
require.NoError(t, err)
|
|
|
|
_, err = writeEvent(ctx, backend, "item4", resourcepb.WatchEvent_ADDED,
|
|
WithNamespace(nsPrefix+"-stats-ns2"),
|
|
WithGroup("group"),
|
|
WithResource("resource1"))
|
|
require.NoError(t, err)
|
|
|
|
_, err = writeEvent(ctx, backend, "item5", resourcepb.WatchEvent_ADDED,
|
|
WithNamespace(nsPrefix+"-stats-ns2"),
|
|
WithGroup("group"),
|
|
WithResource("resource1"))
|
|
require.NoError(t, err)
|
|
|
|
t.Run("Get stats for ns1", func(t *testing.T) {
|
|
nsr := resource.NamespacedResource{
|
|
Namespace: nsPrefix + "-stats-ns1",
|
|
}
|
|
stats, err := backend.GetResourceStats(ctx, nsr, 0)
|
|
require.NoError(t, err)
|
|
require.Len(t, stats, 2)
|
|
|
|
// Sort results for consistent testing
|
|
slices.SortFunc(stats, sortFunc)
|
|
|
|
// Check first resource stats
|
|
require.Equal(t, nsPrefix+"-stats-ns1", stats[0].Namespace)
|
|
require.Equal(t, "group", stats[0].Group)
|
|
require.Equal(t, "resource1", stats[0].Resource)
|
|
require.Equal(t, int64(2), stats[0].Count)
|
|
require.Greater(t, stats[0].ResourceVersion, int64(0))
|
|
|
|
// Check second resource stats
|
|
require.Equal(t, nsPrefix+"-stats-ns1", stats[1].Namespace)
|
|
require.Equal(t, "group", stats[1].Group)
|
|
require.Equal(t, "resource2", stats[1].Resource)
|
|
require.Equal(t, int64(1), stats[1].Count)
|
|
require.Greater(t, stats[1].ResourceVersion, int64(0))
|
|
})
|
|
|
|
t.Run("Get stats for ns2", func(t *testing.T) {
|
|
nsr := resource.NamespacedResource{
|
|
Namespace: nsPrefix + "-stats-ns2",
|
|
}
|
|
stats, err := backend.GetResourceStats(ctx, nsr, 0)
|
|
require.NoError(t, err)
|
|
require.Len(t, stats, 1)
|
|
|
|
require.Equal(t, nsPrefix+"-stats-ns2", stats[0].Namespace)
|
|
require.Equal(t, "group", stats[0].Group)
|
|
require.Equal(t, "resource1", stats[0].Resource)
|
|
require.Equal(t, int64(2), stats[0].Count)
|
|
require.Greater(t, stats[0].ResourceVersion, int64(0))
|
|
})
|
|
|
|
t.Run("Get stats with minimum count", func(t *testing.T) {
|
|
nsr := resource.NamespacedResource{
|
|
Namespace: nsPrefix + "-stats-ns1",
|
|
}
|
|
stats, err := backend.GetResourceStats(ctx, nsr, 1)
|
|
require.NoError(t, err)
|
|
require.Len(t, stats, 1)
|
|
|
|
require.Equal(t, nsPrefix+"-stats-ns1", stats[0].Namespace)
|
|
require.Equal(t, "group", stats[0].Group)
|
|
require.Equal(t, "resource1", stats[0].Resource)
|
|
require.Equal(t, int64(2), stats[0].Count)
|
|
})
|
|
|
|
t.Run("Get stats for non-existent namespace", func(t *testing.T) {
|
|
nsr := resource.NamespacedResource{
|
|
Namespace: "non-existent",
|
|
}
|
|
stats, err := backend.GetResourceStats(ctx, nsr, 0)
|
|
require.NoError(t, err)
|
|
require.Empty(t, stats)
|
|
})
|
|
}
|
|
|
|
func runTestIntegrationBackendWatchWriteEvents(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
|
|
|
|
// Create a few resources before initing the watch
|
|
_, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(nsPrefix+"-watch-ns"))
|
|
require.NoError(t, err)
|
|
|
|
// Start the watch
|
|
stream, err := backend.WatchWriteEvents(ctx)
|
|
require.NoError(t, err)
|
|
|
|
// Create one more event
|
|
_, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(nsPrefix+"-watch-ns"))
|
|
require.NoError(t, err)
|
|
require.Equal(t, "item2", (<-stream).Key.Name)
|
|
|
|
// Should close the stream
|
|
ctx.Cancel()
|
|
|
|
_, ok := <-stream
|
|
require.False(t, ok)
|
|
}
|
|
|
|
func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
|
|
server := newServer(t, backend)
|
|
ns := nsPrefix + "-list-ns"
|
|
// Create a few resources before starting the watch
|
|
rv1, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv1, int64(0))
|
|
rv2, err := writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv2, rv1)
|
|
rv3, err := writeEvent(ctx, backend, "item3", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv3, rv2)
|
|
rv4, err := writeEvent(ctx, backend, "item4", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv4, rv3)
|
|
rv5, err := writeEvent(ctx, backend, "item5", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv5, rv4)
|
|
rv6, err := writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rv2))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv6, rv5)
|
|
rv7, err := writeEvent(ctx, backend, "item3", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rv3))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv7, rv6)
|
|
rv8, err := writeEvent(ctx, backend, "item6", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv8, rv7)
|
|
|
|
t.Run("fetch all latest", func(t *testing.T) {
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 5)
|
|
// should be sorted by key ASC
|
|
require.Contains(t, string(res.Items[0].Value), "item1 ADDED")
|
|
require.Contains(t, string(res.Items[1].Value), "item2 MODIFIED")
|
|
require.Contains(t, string(res.Items[2].Value), "item4 ADDED")
|
|
require.Contains(t, string(res.Items[3].Value), "item5 ADDED")
|
|
require.Contains(t, string(res.Items[4].Value), "item6 ADDED")
|
|
|
|
require.Empty(t, res.NextPageToken)
|
|
})
|
|
|
|
t.Run("fetch all with limit 0", func(t *testing.T) {
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 0,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 5)
|
|
// should be sorted by key ASC
|
|
require.Contains(t, string(res.Items[0].Value), "item1 ADDED")
|
|
require.Contains(t, string(res.Items[1].Value), "item2 MODIFIED")
|
|
require.Contains(t, string(res.Items[2].Value), "item4 ADDED")
|
|
require.Contains(t, string(res.Items[3].Value), "item5 ADDED")
|
|
require.Contains(t, string(res.Items[4].Value), "item6 ADDED")
|
|
|
|
require.Empty(t, res.NextPageToken)
|
|
})
|
|
|
|
t.Run("fetch all latest with pagination", func(t *testing.T) {
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 3)
|
|
require.Contains(t, string(res.Items[0].Value), "item1 ADDED")
|
|
require.Contains(t, string(res.Items[1].Value), "item2 MODIFIED")
|
|
require.Contains(t, string(res.Items[2].Value), "item4 ADDED")
|
|
require.NotEmpty(t, res.NextPageToken)
|
|
|
|
res, err = server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
NextPageToken: res.NextPageToken,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Contains(t, string(res.Items[0].Value), "item5 ADDED")
|
|
require.Contains(t, string(res.Items[1].Value), "item6 ADDED")
|
|
require.Len(t, res.Items, 2)
|
|
require.Empty(t, res.NextPageToken)
|
|
})
|
|
|
|
t.Run("list at revision", func(t *testing.T) {
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
ResourceVersion: rv4,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 4)
|
|
require.Contains(t, string(res.Items[0].Value), "item1 ADDED")
|
|
require.Contains(t, string(res.Items[1].Value), "item2 ADDED")
|
|
require.Contains(t, string(res.Items[2].Value), "item3 ADDED")
|
|
require.Contains(t, string(res.Items[3].Value), "item4 ADDED")
|
|
require.Empty(t, res.NextPageToken)
|
|
})
|
|
|
|
t.Run("list at revision with pagination", func(t *testing.T) {
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
ResourceVersion: rv7,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 3)
|
|
require.Contains(t, string(res.Items[0].Value), "item1 ADDED")
|
|
require.Contains(t, string(res.Items[1].Value), "item2 MODIFIED")
|
|
require.Contains(t, string(res.Items[2].Value), "item4 ADDED")
|
|
require.NotEmpty(t, res.NextPageToken)
|
|
|
|
res, err = server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
ResourceVersion: rv7,
|
|
NextPageToken: res.NextPageToken,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 1)
|
|
require.Contains(t, string(res.Items[0].Value), "item5 ADDED")
|
|
require.Empty(t, res.NextPageToken)
|
|
})
|
|
}
|
|
|
|
func runTestIntegrationBackendListModifiedSince(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
|
ns := nsPrefix + "-history-ns"
|
|
rvCreated, _ := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.Greater(t, rvCreated, int64(0))
|
|
rvUpdated, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rvCreated))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvUpdated, rvCreated)
|
|
rvDeleted, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rvUpdated))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvDeleted, rvUpdated)
|
|
|
|
t.Run("will list latest modified event when resource has multiple events", func(t *testing.T) {
|
|
key := resource.NamespacedResource{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
}
|
|
latestRv, seq := backend.ListModifiedSince(ctx, key, rvCreated)
|
|
require.Greater(t, latestRv, rvCreated)
|
|
|
|
counter := 0
|
|
for res, err := range seq {
|
|
require.NoError(t, err)
|
|
require.Equal(t, rvDeleted, res.ResourceVersion)
|
|
counter++
|
|
}
|
|
require.Equal(t, 1, counter) // only one event should be returned
|
|
})
|
|
|
|
t.Run("no events if none after the given resource version", func(t *testing.T) {
|
|
key := resource.NamespacedResource{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
}
|
|
latestRv, seq := backend.ListModifiedSince(ctx, key, rvDeleted)
|
|
require.GreaterOrEqual(t, latestRv, rvDeleted)
|
|
|
|
isEmpty(t, seq)
|
|
})
|
|
|
|
t.Run("no events for subsequent listModifiedSince calls", func(t *testing.T) {
|
|
key := resource.NamespacedResource{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
}
|
|
latestRv1, seq := backend.ListModifiedSince(ctx, key, rvDeleted)
|
|
require.GreaterOrEqual(t, latestRv1, rvDeleted)
|
|
isEmpty(t, seq)
|
|
|
|
latestRv2, seq := backend.ListModifiedSince(ctx, key, latestRv1)
|
|
require.Equal(t, latestRv1, latestRv2)
|
|
isEmpty(t, seq)
|
|
})
|
|
|
|
t.Run("will only return modified events for the given key", func(t *testing.T) {
|
|
key := resource.NamespacedResource{
|
|
Namespace: "other-ns",
|
|
Group: "group",
|
|
Resource: "resource",
|
|
}
|
|
|
|
// Write an event for another tenant for the same resource
|
|
rvCreatedOtherTenant, err := writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace("other-ns"))
|
|
require.NoError(t, err)
|
|
|
|
latestRv, seq := backend.ListModifiedSince(ctx, key, rvCreated)
|
|
require.Greater(t, latestRv, rvCreated)
|
|
|
|
counter := 0
|
|
for res, err := range seq {
|
|
require.NoError(t, err)
|
|
require.Equal(t, rvCreatedOtherTenant, res.ResourceVersion)
|
|
require.Equal(t, key.Namespace, res.Key.Namespace)
|
|
counter++
|
|
}
|
|
require.Equal(t, 1, counter) // only one event should be returned
|
|
})
|
|
|
|
t.Run("will order events by resource version ascending and name descending", func(t *testing.T) {
|
|
key := resource.NamespacedResource{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
}
|
|
|
|
rvCreated1, _ := writeEvent(ctx, backend, "cItem", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
rvCreated2, _ := writeEvent(ctx, backend, "aItem", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
rvCreated3, _ := writeEvent(ctx, backend, "bItem", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
|
|
latestRv, seq := backend.ListModifiedSince(ctx, key, rvCreated1-1)
|
|
require.Greater(t, latestRv, rvCreated3)
|
|
|
|
counter := 0
|
|
names := []string{"aItem", "bItem", "cItem"}
|
|
rvs := []int64{rvCreated2, rvCreated3, rvCreated1}
|
|
for res, err := range seq {
|
|
require.NoError(t, err)
|
|
require.Equal(t, key.Namespace, res.Key.Namespace)
|
|
require.Equal(t, names[counter], res.Key.Name)
|
|
require.Equal(t, rvs[counter], res.ResourceVersion)
|
|
counter++
|
|
}
|
|
require.Equal(t, 3, counter)
|
|
})
|
|
}
|
|
|
|
func isEmpty(t *testing.T, seq iter.Seq2[*resource.ModifiedResource, error]) {
|
|
counter := 0
|
|
for range seq {
|
|
counter++
|
|
}
|
|
require.Equal(t, 0, counter)
|
|
}
|
|
|
|
func runTestIntegrationBackendListHistory(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
|
server := newServer(t, backend)
|
|
ns := nsPrefix + "-history-ns"
|
|
rv1, _ := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.Greater(t, rv1, int64(0))
|
|
|
|
// add 5 events for item1 - should be saved to history
|
|
rvHistory1, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rv1))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvHistory1, rv1)
|
|
rvHistory2, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rvHistory1))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvHistory2, rvHistory1)
|
|
rvHistory3, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rvHistory2))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvHistory3, rvHistory2)
|
|
rvHistory4, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rvHistory3))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvHistory4, rvHistory3)
|
|
rvHistory5, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rvHistory4))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvHistory5, rvHistory4)
|
|
|
|
t.Run("fetch history with different version matching", func(t *testing.T) {
|
|
baseKey := &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "item1",
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
request *resourcepb.ListRequest
|
|
expectedVersions []int64
|
|
expectedValues []string
|
|
minExpectedHeadRV int64
|
|
expectedContinueRV int64
|
|
expectedSortAsc bool
|
|
}{
|
|
{
|
|
name: "NotOlderThan with rv1 (ASC order)",
|
|
request: &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
ResourceVersion: rv1,
|
|
VersionMatchV2: resourcepb.ResourceVersionMatchV2_NotOlderThan,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: baseKey,
|
|
},
|
|
},
|
|
expectedVersions: []int64{rv1, rvHistory1, rvHistory2},
|
|
expectedValues: []string{"item1 ADDED", "item1 MODIFIED", "item1 MODIFIED"},
|
|
minExpectedHeadRV: rvHistory2,
|
|
expectedContinueRV: rvHistory2,
|
|
expectedSortAsc: true,
|
|
},
|
|
{
|
|
name: "NotOlderThan with rv=0 (ASC order)",
|
|
request: &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
ResourceVersion: 0,
|
|
VersionMatchV2: resourcepb.ResourceVersionMatchV2_NotOlderThan,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: baseKey,
|
|
},
|
|
},
|
|
expectedVersions: []int64{rv1, rvHistory1, rvHistory2},
|
|
expectedValues: []string{"item1 ADDED", "item1 MODIFIED", "item1 MODIFIED"},
|
|
minExpectedHeadRV: rvHistory2,
|
|
expectedContinueRV: rvHistory2,
|
|
expectedSortAsc: true,
|
|
},
|
|
{
|
|
name: "ResourceVersionMatch_Unset (DESC order)",
|
|
request: &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: baseKey,
|
|
},
|
|
},
|
|
expectedVersions: []int64{rvHistory5, rvHistory4, rvHistory3},
|
|
expectedValues: []string{"item1 MODIFIED", "item1 MODIFIED", "item1 MODIFIED"},
|
|
minExpectedHeadRV: rvHistory5,
|
|
expectedContinueRV: rvHistory3,
|
|
expectedSortAsc: false,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
res, err := server.List(ctx, tc.request)
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 3)
|
|
|
|
// Check versions and values match expectations
|
|
for i := 0; i < 3; i++ {
|
|
require.Equal(t, tc.expectedVersions[i], res.Items[i].ResourceVersion)
|
|
require.Contains(t, string(res.Items[i].Value), tc.expectedValues[i])
|
|
}
|
|
|
|
// Check resource version in response
|
|
require.GreaterOrEqual(t, res.ResourceVersion, tc.minExpectedHeadRV)
|
|
|
|
// Check continue token
|
|
require.NotEmpty(t, res.NextPageToken)
|
|
})
|
|
}
|
|
|
|
// Test pagination for NotOlderThan (second page)
|
|
t.Run("second page with NotOlderThan", func(t *testing.T) {
|
|
// Get first page
|
|
firstRequest := &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
ResourceVersion: rv1,
|
|
VersionMatchV2: resourcepb.ResourceVersionMatchV2_NotOlderThan,
|
|
Options: &resourcepb.ListOptions{Key: baseKey},
|
|
}
|
|
firstPageRes, err := server.List(ctx, firstRequest)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, firstPageRes.NextPageToken)
|
|
|
|
// Get second page
|
|
secondPageRes, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 3,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
ResourceVersion: rv1,
|
|
VersionMatchV2: resourcepb.ResourceVersionMatchV2_NotOlderThan,
|
|
NextPageToken: firstPageRes.NextPageToken,
|
|
Options: &resourcepb.ListOptions{Key: baseKey},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, secondPageRes.Error)
|
|
require.Len(t, secondPageRes.Items, 3)
|
|
require.Empty(t, secondPageRes.NextPageToken)
|
|
|
|
// Second page should continue in ascending order
|
|
expectedRVs := []int64{rvHistory3, rvHistory4, rvHistory5}
|
|
for i, expectedRV := range expectedRVs {
|
|
require.Equal(t, expectedRV, secondPageRes.Items[i].ResourceVersion)
|
|
require.Contains(t, string(secondPageRes.Items[i].Value), "item1 MODIFIED")
|
|
}
|
|
})
|
|
|
|
// Test with limit=0 (should return all items)
|
|
t.Run("fetch all history with limit 0", func(t *testing.T) {
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Limit: 0,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: baseKey,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 6) // Should return all 6 history items (1 ADDED + 5 MODIFIED)
|
|
|
|
// Should be in descending order (default for history)
|
|
require.Equal(t, rvHistory5, res.Items[0].ResourceVersion)
|
|
require.Equal(t, rvHistory4, res.Items[1].ResourceVersion)
|
|
require.Equal(t, rvHistory3, res.Items[2].ResourceVersion)
|
|
require.Equal(t, rvHistory2, res.Items[3].ResourceVersion)
|
|
require.Equal(t, rvHistory1, res.Items[4].ResourceVersion)
|
|
require.Equal(t, rv1, res.Items[5].ResourceVersion)
|
|
|
|
require.Empty(t, res.NextPageToken)
|
|
})
|
|
})
|
|
|
|
t.Run("fetch second page of history at revision", func(t *testing.T) {
|
|
continueToken := &resource.ContinueToken{
|
|
ResourceVersion: rvHistory3,
|
|
SortAscending: false,
|
|
}
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
NextPageToken: continueToken.String(),
|
|
Limit: 2,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "item1",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 2)
|
|
t.Log(res.Items)
|
|
require.Contains(t, string(res.Items[0].Value), "item1 MODIFIED")
|
|
require.Equal(t, rvHistory2, res.Items[0].ResourceVersion)
|
|
require.Contains(t, string(res.Items[1].Value), "item1 MODIFIED")
|
|
require.Equal(t, rvHistory1, res.Items[1].ResourceVersion)
|
|
})
|
|
|
|
t.Run("paginated history with NotOlderThan returns items in ascending order", func(t *testing.T) {
|
|
// Create 10 versions of a resource to test pagination
|
|
ns2 := nsPrefix + "-ns2"
|
|
resourceKey := &resourcepb.ResourceKey{
|
|
Namespace: ns2,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "paged-item",
|
|
}
|
|
|
|
var resourceVersions []int64
|
|
|
|
// First create the initial resource
|
|
initialRV, err := writeEvent(ctx, backend, "paged-item", resourcepb.WatchEvent_ADDED, WithNamespace(ns2))
|
|
require.NoError(t, err)
|
|
resourceVersions = append(resourceVersions, initialRV)
|
|
|
|
// Create 9 more versions with modifications
|
|
rv := initialRV
|
|
for i := 0; i < 9; i++ {
|
|
rv, err = writeEvent(ctx, backend, "paged-item", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns2, rv))
|
|
require.NoError(t, err)
|
|
resourceVersions = append(resourceVersions, rv)
|
|
}
|
|
|
|
// Now we should have 10 versions total (1 ADDED + 9 MODIFIED)
|
|
require.Len(t, resourceVersions, 10)
|
|
|
|
// Test pagination with limit of 3 and NotOlderThan, starting from the beginning
|
|
pages := []struct {
|
|
pageNumber int
|
|
pageSize int
|
|
startToken string
|
|
}{
|
|
{pageNumber: 1, pageSize: 3, startToken: ""},
|
|
{pageNumber: 2, pageSize: 3, startToken: ""}, // Will be set in the test
|
|
{pageNumber: 3, pageSize: 3, startToken: ""}, // Will be set in the test
|
|
{pageNumber: 4, pageSize: 1, startToken: ""}, // Will be set in the test - last page with remaining item
|
|
}
|
|
|
|
var allItems []*resourcepb.ResourceWrapper
|
|
|
|
// Request first page with NotOlderThan and ResourceVersion=0 (should start from oldest)
|
|
for i, page := range pages {
|
|
req := &resourcepb.ListRequest{
|
|
Limit: int64(page.pageSize),
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
ResourceVersion: 0,
|
|
VersionMatchV2: resourcepb.ResourceVersionMatchV2_NotOlderThan,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: resourceKey,
|
|
},
|
|
}
|
|
|
|
if i > 0 {
|
|
// For subsequent pages, use the continue token from the previous page
|
|
req.NextPageToken = pages[i].startToken
|
|
}
|
|
|
|
res, err := server.List(ctx, req)
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
|
|
// First 3 pages should have exactly pageSize items
|
|
if i < 3 {
|
|
require.Len(t, res.Items, page.pageSize, "Page %d should have %d items", i+1, page.pageSize)
|
|
} else {
|
|
// Last page should have 1 item (10 items total with 3+3+3+1 distribution)
|
|
require.Len(t, res.Items, 1, "Last page should have 1 item")
|
|
}
|
|
|
|
// Save continue token for next page if not the last page
|
|
if i < len(pages)-1 {
|
|
pages[i+1].startToken = res.NextPageToken
|
|
require.NotEmpty(t, res.NextPageToken, "Should have continue token for page %d", i+1)
|
|
} else {
|
|
// Last page should not have a continue token
|
|
require.Empty(t, res.NextPageToken, "Last page should not have continue token")
|
|
}
|
|
|
|
// Add items to our collection
|
|
allItems = append(allItems, res.Items...)
|
|
|
|
// Verify all items in current page are in ascending order
|
|
for j := 1; j < len(res.Items); j++ {
|
|
require.Less(t, res.Items[j-1].ResourceVersion, res.Items[j].ResourceVersion,
|
|
"Items within page %d should be in ascending order", i+1)
|
|
}
|
|
|
|
// For pages after the first, verify first item of current page is greater than last item of previous page
|
|
if i > 0 && len(allItems) > page.pageSize {
|
|
prevPageLastIdx := len(allItems) - len(res.Items) - 1
|
|
currentPageFirstIdx := len(allItems) - len(res.Items)
|
|
require.Greater(t, allItems[currentPageFirstIdx].ResourceVersion, allItems[prevPageLastIdx].ResourceVersion,
|
|
"First item of page %d should have higher RV than last item of page %d", i+1, i)
|
|
}
|
|
}
|
|
|
|
// Verify we got all 10 items
|
|
require.Len(t, allItems, 10, "Should have retrieved all 10 items")
|
|
|
|
// Verify all items are in ascending order of resource version
|
|
for i := 1; i < len(allItems); i++ {
|
|
require.Less(t, allItems[i-1].ResourceVersion, allItems[i].ResourceVersion,
|
|
"All items should be in ascending order of resource version")
|
|
}
|
|
|
|
// Verify the first item is the initial ADDED event
|
|
require.Equal(t, initialRV, allItems[0].ResourceVersion, "First item should be the initial ADDED event")
|
|
require.Contains(t, string(allItems[0].Value), "paged-item ADDED")
|
|
|
|
// Verify all other items are MODIFIED events and correspond to our recorded resource versions
|
|
for i := 1; i < len(allItems); i++ {
|
|
require.Contains(t, string(allItems[i].Value), "paged-item MODIFIED")
|
|
require.Equal(t, resourceVersions[i], allItems[i].ResourceVersion)
|
|
}
|
|
})
|
|
|
|
t.Run("fetch history with deleted item", func(t *testing.T) {
|
|
// Create a resource and delete it
|
|
rv, err := writeEvent(ctx, backend, "deleted-item", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
rvDeleted, err := writeEvent(ctx, backend, "deleted-item", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rv))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvDeleted, rv)
|
|
|
|
// Fetch history for the deleted item
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "deleted-item",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 0)
|
|
})
|
|
|
|
t.Run("fetch history with recreated item", func(t *testing.T) {
|
|
// Create a resource and delete it
|
|
rv, err := writeEvent(ctx, backend, "deleted-item", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
rvDeleted, err := writeEvent(ctx, backend, "deleted-item", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rv))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvDeleted, rv)
|
|
|
|
// Create a few more versions after deletion
|
|
rv1, err := writeEvent(ctx, backend, "deleted-item", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv1, rvDeleted)
|
|
rv2, err := writeEvent(ctx, backend, "deleted-item", resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, rv1))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv2, rv1)
|
|
|
|
// Fetch history for the deleted item
|
|
res, err := server.List(ctx, &resourcepb.ListRequest{
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "deleted-item",
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
require.Len(t, res.Items, 2)
|
|
require.Contains(t, string(res.Items[0].Value), "deleted-item MODIFIED")
|
|
require.Equal(t, rv2, res.Items[0].ResourceVersion)
|
|
require.Contains(t, string(res.Items[1].Value), "deleted-item ADDED")
|
|
require.Equal(t, rv1, res.Items[1].ResourceVersion)
|
|
})
|
|
}
|
|
|
|
func runTestIntegrationBackendListHistoryErrorReporting(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
|
server := newServer(t, backend)
|
|
|
|
ns := nsPrefix + "-short"
|
|
const (
|
|
name = "it1"
|
|
group = "group"
|
|
resourceName = "resource"
|
|
)
|
|
|
|
start := time.Now()
|
|
origRv, _ := writeEvent(ctx, backend, name, resourcepb.WatchEvent_ADDED, WithNamespace(ns), WithGroup(group), WithResource(resourceName))
|
|
require.Greater(t, origRv, int64(0))
|
|
|
|
const events = 500
|
|
prevRv := origRv
|
|
for range events {
|
|
rv, err := writeEvent(ctx, backend, name, resourcepb.WatchEvent_MODIFIED, WithNamespaceAndRV(ns, prevRv), WithGroup(group), WithResource(resourceName))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv, prevRv)
|
|
prevRv = rv
|
|
}
|
|
t.Log("added events in ", time.Since(start))
|
|
|
|
req := &resourcepb.ListRequest{
|
|
Limit: 2 * events,
|
|
Source: resourcepb.ListRequest_HISTORY,
|
|
ResourceVersion: origRv,
|
|
VersionMatchV2: resourcepb.ResourceVersionMatchV2_NotOlderThan,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: group,
|
|
Resource: resourceName,
|
|
Name: name,
|
|
},
|
|
},
|
|
}
|
|
|
|
shortContext, cancel := context.WithTimeout(ctx, 1*time.Microsecond)
|
|
defer cancel()
|
|
|
|
res, err := server.List(shortContext, req)
|
|
// We expect context deadline error, but it may be reported as a res.Error object.
|
|
t.Log("list error:", err)
|
|
if res != nil {
|
|
t.Log("iterator error:", res.Error)
|
|
t.Log("numItems:", len(res.Items))
|
|
}
|
|
require.True(t, err != nil || (res != nil && res.Error != nil))
|
|
}
|
|
|
|
func runTestIntegrationBlobSupport(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
|
|
server := newServer(t, backend)
|
|
store, ok := backend.(resource.BlobSupport)
|
|
require.True(t, ok)
|
|
ns := nsPrefix + "-ns1"
|
|
|
|
t.Run("put and fetch blob", func(t *testing.T) {
|
|
key := &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "ggg",
|
|
Resource: "rrr",
|
|
Name: "nnn",
|
|
}
|
|
b1, err := server.PutBlob(ctx, &resourcepb.PutBlobRequest{
|
|
Resource: key,
|
|
Method: resourcepb.PutBlobRequest_GRPC,
|
|
ContentType: "plain/text",
|
|
Value: []byte("hello 11111"),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, b1.Error)
|
|
require.Equal(t, "c894ae57bd227b8f8c63f38a2ddf458b", b1.Hash)
|
|
|
|
b2, err := server.PutBlob(ctx, &resourcepb.PutBlobRequest{
|
|
Resource: key,
|
|
Method: resourcepb.PutBlobRequest_GRPC,
|
|
ContentType: "plain/text",
|
|
Value: []byte("hello 22222"), // the most recent
|
|
})
|
|
require.NoError(t, err)
|
|
require.Nil(t, b2.Error)
|
|
require.Equal(t, "b0da48de4ff92e0ad0d836de4d746937", b2.Hash)
|
|
|
|
// Check that we can still access both values
|
|
found, err := store.GetResourceBlob(ctx, key, &utils.BlobInfo{UID: b1.Uid}, true)
|
|
require.NoError(t, err)
|
|
require.Contains(t, string(found.Value), "hello 11111")
|
|
|
|
found, err = store.GetResourceBlob(ctx, key, &utils.BlobInfo{UID: b2.Uid}, true)
|
|
require.NoError(t, err)
|
|
require.Contains(t, string(found.Value), "hello 22222")
|
|
|
|
// Save a resource with annotation
|
|
obj := &unstructured.Unstructured{}
|
|
meta, err := utils.MetaAccessor(obj)
|
|
require.NoError(t, err)
|
|
meta.SetBlob(&utils.BlobInfo{UID: b2.Uid, Hash: b1.Hash})
|
|
meta.SetName(key.Name)
|
|
meta.SetNamespace(key.Namespace)
|
|
obj.SetAPIVersion(key.Group + "/v1")
|
|
obj.SetKind("Test")
|
|
val, err := obj.MarshalJSON()
|
|
require.NoError(t, err)
|
|
out, err := server.Create(ctx, &resourcepb.CreateRequest{Key: key, Value: val})
|
|
require.NoError(t, err)
|
|
require.Nil(t, out.Error)
|
|
require.True(t, out.ResourceVersion > 0)
|
|
|
|
// The server (not store!) will lookup the saved annotation and return the correct payload
|
|
res, err := server.GetBlob(ctx, &resourcepb.GetBlobRequest{Resource: key})
|
|
require.NoError(t, err)
|
|
require.Nil(t, out.Error)
|
|
require.Contains(t, string(res.Value), "hello 22222")
|
|
|
|
// But we can still get an older version with an explicit UID
|
|
res, err = server.GetBlob(ctx, &resourcepb.GetBlobRequest{Resource: key, Uid: b1.Uid})
|
|
require.NoError(t, err)
|
|
require.Nil(t, out.Error)
|
|
require.Contains(t, string(res.Value), "hello 11111")
|
|
})
|
|
}
|
|
|
|
func runTestIntegrationBackendCreateNewResource(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := types.WithAuthInfo(t.Context(), authn.NewAccessTokenAuthInfo(authn.Claims[authn.AccessTokenClaims]{
|
|
Claims: jwt.Claims{
|
|
Subject: "testuser",
|
|
},
|
|
Rest: authn.AccessTokenClaims{},
|
|
}))
|
|
|
|
server := newServer(t, backend)
|
|
ns := nsPrefix + "-create-resource"
|
|
ctx = request.WithNamespace(ctx, ns)
|
|
|
|
request := &resourcepb.CreateRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "test.grafana",
|
|
Resource: "tests",
|
|
Name: "test",
|
|
},
|
|
Value: []byte(`{"apiVersion":"test.grafana/v0alpha1","kind":"Test","metadata":{"name":"test","namespace":"` + ns + `","uid":"test-uid-123"}}`),
|
|
}
|
|
|
|
response, err := server.Create(ctx, request)
|
|
require.NoError(t, err, "create resource")
|
|
require.Nil(t, response.Error, "create resource response.Error")
|
|
|
|
t.Run("gracefully handles resource already exists error", func(t *testing.T) {
|
|
response, err := server.Create(ctx, request)
|
|
require.NoError(t, err, "create resource")
|
|
require.NotNil(t, response.GetError(), "create resource response.Error")
|
|
assert.Equal(t, int32(http.StatusConflict), response.GetError().GetCode(), "create resource response.Error.Code")
|
|
assert.Equal(t, string(metav1.StatusReasonAlreadyExists), response.GetError().GetReason(), "create resource response.Error.Reason")
|
|
t.Logf("Error: %v", response.GetError()) // only prints on failure, so this is fine
|
|
})
|
|
}
|
|
|
|
// WriteEventOption is a function that modifies WriteEventOptions
|
|
type WriteEventOption func(*WriteEventOptions)
|
|
|
|
// WithNamespace sets the namespace for the write event
|
|
func WithNamespaceAndRV(namespace string, rv int64) WriteEventOption {
|
|
return func(o *WriteEventOptions) {
|
|
o.Namespace = namespace
|
|
o.PreviousRV = rv
|
|
}
|
|
}
|
|
|
|
// WithNamespace sets the namespace for the write event
|
|
func WithNamespace(namespace string) WriteEventOption {
|
|
return func(o *WriteEventOptions) {
|
|
o.Namespace = namespace
|
|
}
|
|
}
|
|
|
|
// WithGroup sets the group for the write event
|
|
func WithGroup(group string) WriteEventOption {
|
|
return func(o *WriteEventOptions) {
|
|
o.Group = group
|
|
}
|
|
}
|
|
|
|
// WithResource sets the resource for the write event
|
|
func WithResource(resource string) WriteEventOption {
|
|
return func(o *WriteEventOptions) {
|
|
o.Resource = resource
|
|
}
|
|
}
|
|
|
|
// WithFolder sets the folder for the write event
|
|
func WithFolder(folder string) WriteEventOption {
|
|
return func(o *WriteEventOptions) {
|
|
o.Folder = folder
|
|
}
|
|
}
|
|
|
|
// WithValue sets the value for the write event
|
|
func WithValue(value string) WriteEventOption {
|
|
return func(o *WriteEventOptions) {
|
|
u := unstructured.Unstructured{
|
|
Object: map[string]any{
|
|
"apiVersion": o.Group + "/v1",
|
|
"kind": o.Resource,
|
|
"metadata": map[string]any{
|
|
"name": "name",
|
|
"namespace": "ns",
|
|
},
|
|
"spec": map[string]any{
|
|
"value": value,
|
|
},
|
|
},
|
|
}
|
|
o.Value, _ = u.MarshalJSON()
|
|
}
|
|
}
|
|
|
|
type WriteEventOptions struct {
|
|
Namespace string
|
|
Group string
|
|
Resource string
|
|
Folder string
|
|
Value []byte
|
|
PreviousRV int64
|
|
}
|
|
|
|
func writeEvent(ctx context.Context, store resource.StorageBackend, name string, action resourcepb.WatchEvent_Type, opts ...WriteEventOption) (int64, error) {
|
|
// Default options
|
|
options := WriteEventOptions{
|
|
Namespace: "namespace",
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Folder: "folderuid",
|
|
}
|
|
|
|
// Apply options
|
|
for _, opt := range opts {
|
|
opt(&options)
|
|
}
|
|
if options.Value == nil {
|
|
u := unstructured.Unstructured{
|
|
Object: map[string]any{
|
|
"apiVersion": options.Group + "/v1",
|
|
"kind": options.Resource,
|
|
"metadata": map[string]any{
|
|
"name": name,
|
|
"namespace": options.Namespace,
|
|
},
|
|
"spec": map[string]any{
|
|
"value": name + " " + resourcepb.WatchEvent_Type_name[int32(action)],
|
|
},
|
|
},
|
|
}
|
|
options.Value, _ = u.MarshalJSON()
|
|
}
|
|
|
|
res := &unstructured.Unstructured{
|
|
Object: map[string]any{},
|
|
}
|
|
meta, err := utils.MetaAccessor(res)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
meta.SetFolder(options.Folder)
|
|
|
|
event := resource.WriteEvent{
|
|
Type: action,
|
|
Value: options.Value,
|
|
GUID: uuid.New().String(),
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: options.Namespace,
|
|
Group: options.Group,
|
|
Resource: options.Resource,
|
|
Name: name,
|
|
},
|
|
PreviousRV: options.PreviousRV,
|
|
}
|
|
switch action {
|
|
case resourcepb.WatchEvent_DELETED:
|
|
event.ObjectOld = meta
|
|
|
|
obj, err := utils.MetaAccessor(res)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
now := metav1.Now()
|
|
obj.SetDeletionTimestamp(&now)
|
|
obj.SetUpdatedTimestamp(&now.Time)
|
|
obj.SetManagedFields(nil)
|
|
obj.SetFinalizers(nil)
|
|
obj.SetGeneration(utils.DeletedGeneration)
|
|
obj.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
|
|
event.Object = obj
|
|
case resourcepb.WatchEvent_ADDED:
|
|
event.Object = meta
|
|
case resourcepb.WatchEvent_MODIFIED:
|
|
event.Object = meta //
|
|
event.ObjectOld = meta
|
|
default:
|
|
panic(fmt.Sprintf("invalid action: %s", action))
|
|
}
|
|
return store.WriteEvent(ctx, event)
|
|
}
|
|
|
|
func newServer(t *testing.T, b resource.StorageBackend) resource.ResourceServer {
|
|
t.Helper()
|
|
|
|
server, err := resource.NewResourceServer(resource.ResourceServerOptions{
|
|
Backend: b,
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, server)
|
|
|
|
return server
|
|
}
|
|
|
|
func runTestIntegrationBackendTrash(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
|
server := newServer(t, backend)
|
|
ns := nsPrefix + "-ns-trash"
|
|
|
|
// item1 deleted with multiple history events
|
|
rv1, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv1, int64(0))
|
|
rvDelete1, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rv1))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvDelete1, rv1)
|
|
|
|
// item2 deleted and recreated, should not be returned in trash
|
|
rv2, err := writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv2, int64(0))
|
|
rvDelete3, err := writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_DELETED, WithNamespaceAndRV(ns, rv2))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rvDelete3, rv2)
|
|
rv3, err := writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv3, int64(0))
|
|
|
|
tests := []struct {
|
|
name string
|
|
request *resourcepb.ListRequest
|
|
expectedVersions []int64
|
|
expectedValues []string
|
|
minExpectedHeadRV int64
|
|
expectedContinueRV int64
|
|
expectedSortAsc bool
|
|
}{
|
|
{
|
|
name: "returns the latest delete event",
|
|
request: &resourcepb.ListRequest{
|
|
Source: resourcepb.ListRequest_TRASH,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "item1",
|
|
},
|
|
},
|
|
},
|
|
expectedVersions: []int64{rvDelete1},
|
|
expectedValues: []string{"item1 DELETED"},
|
|
minExpectedHeadRV: rvDelete1,
|
|
expectedContinueRV: rvDelete1,
|
|
expectedSortAsc: false,
|
|
},
|
|
{
|
|
name: "does not return a version in the resource table",
|
|
request: &resourcepb.ListRequest{
|
|
Source: resourcepb.ListRequest_TRASH,
|
|
Options: &resourcepb.ListOptions{
|
|
Key: &resourcepb.ResourceKey{
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
Name: "item2",
|
|
},
|
|
},
|
|
},
|
|
expectedVersions: []int64{},
|
|
expectedValues: []string{},
|
|
minExpectedHeadRV: rv3,
|
|
expectedContinueRV: rv3,
|
|
expectedSortAsc: false,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
res, err := server.List(ctx, tc.request)
|
|
require.NoError(t, err)
|
|
require.Nil(t, res.Error)
|
|
expectedItemCount := len(tc.expectedVersions)
|
|
require.Len(t, res.Items, expectedItemCount)
|
|
for i := 0; i < expectedItemCount; i++ {
|
|
require.Equal(t, tc.expectedVersions[i], res.Items[i].ResourceVersion)
|
|
require.Contains(t, string(res.Items[i].Value), tc.expectedValues[i])
|
|
}
|
|
require.GreaterOrEqual(t, res.ResourceVersion, tc.minExpectedHeadRV)
|
|
})
|
|
}
|
|
}
|
|
|
|
func runTestIntegrationGetResourceLastImportTime(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
bulk, ok := backend.(resource.BulkProcessingBackend)
|
|
if !ok {
|
|
// This test is only enabled for backends that DO support bulk import. If the backend does not support
|
|
// bulk import but has this test enabled, that's a bug.
|
|
t.Fatal("backend does not support bulk import")
|
|
}
|
|
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
|
|
|
t.Run("no imported times by default", func(t *testing.T) {
|
|
res := collectLastImportedTimes(t, backend, ctx)
|
|
require.Empty(t, res)
|
|
})
|
|
|
|
t.Run("last imported time after bulk import", func(t *testing.T) {
|
|
ns := nsPrefix + "-import"
|
|
|
|
collections := []*resourcepb.ResourceKey{
|
|
{Namespace: ns, Group: "dashboards", Resource: "dashboard"},
|
|
{Namespace: ns, Group: "folders", Resource: "folder"},
|
|
}
|
|
|
|
bulkRequests := []*resourcepb.BulkRequest{
|
|
{
|
|
Key: &resourcepb.ResourceKey{Namespace: ns, Group: "dashboards", Resource: "dashboard", Name: "test"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
},
|
|
{
|
|
Key: &resourcepb.ResourceKey{Namespace: ns, Group: "dashboards", Resource: "dashboard", Name: "test2"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
},
|
|
{
|
|
Key: &resourcepb.ResourceKey{Namespace: ns, Group: "folders", Resource: "folder", Name: "test2"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
},
|
|
}
|
|
|
|
resp := bulk.ProcessBulk(ctx, resource.BulkSettings{
|
|
Collection: collections,
|
|
RebuildCollection: true,
|
|
}, toBulkIterator(bulkRequests))
|
|
require.Nil(t, resp.Error)
|
|
|
|
result := collectLastImportedTimes(t, backend, ctx)
|
|
require.Len(t, result, len(collections))
|
|
|
|
now := time.Now()
|
|
|
|
for _, r := range collections {
|
|
nsr := resource.NamespacedResource{Namespace: r.Namespace, Group: r.Group, Resource: r.Resource}
|
|
lastImported, ok := result[nsr]
|
|
require.True(t, ok, "resource not found: %s", nsr.String())
|
|
require.True(t, lastImported.After(now.Add(-5*time.Second)), "last imported time is not recent")
|
|
}
|
|
})
|
|
|
|
t.Run("last imported time after multiple bulk imports", func(t *testing.T) {
|
|
ns1 := nsPrefix + "-import1"
|
|
collections1 := []*resourcepb.ResourceKey{
|
|
{Namespace: ns1, Group: "dashboards", Resource: "dashboard"},
|
|
{Namespace: ns1, Group: "folders", Resource: "folder"},
|
|
}
|
|
bulkRequests1 := []*resourcepb.BulkRequest{{
|
|
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "dashboards", Resource: "dashboard", Name: "test"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
}, {
|
|
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "dashboards", Resource: "dashboard", Name: "test2"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
}, {
|
|
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "folders", Resource: "folder", Name: "test2"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
}}
|
|
|
|
resp1 := bulk.ProcessBulk(ctx, resource.BulkSettings{
|
|
Collection: collections1,
|
|
RebuildCollection: true,
|
|
}, toBulkIterator(bulkRequests1))
|
|
require.Nil(t, resp1.Error)
|
|
|
|
firstImport := time.Now()
|
|
|
|
const delta = 5 * time.Second
|
|
// Verify that last imported times are combination of both bulk imports
|
|
result1 := collectLastImportedTimes(t, backend, ctx)
|
|
require.WithinDuration(t, result1[resource.NamespacedResource{Namespace: ns1, Group: "dashboards", Resource: "dashboard"}], firstImport, delta)
|
|
require.WithinDuration(t, result1[resource.NamespacedResource{Namespace: ns1, Group: "folders", Resource: "folder"}], firstImport, delta)
|
|
|
|
// Do another bulk import, without overwriting existing resources. We import into ns1-dashboards (same as before),
|
|
// and new ns2-folders. ns1-folders is unchanged.
|
|
ns2 := nsPrefix + "-import2"
|
|
collections2 := []*resourcepb.ResourceKey{
|
|
{Namespace: ns1, Group: "dashboards", Resource: "dashboard"}, // Import MORE dashboards into ns1
|
|
{Namespace: ns2, Group: "folders", Resource: "folder"},
|
|
}
|
|
bulkRequests2 := []*resourcepb.BulkRequest{{
|
|
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "dashboards", Resource: "dashboard", Name: "new-test"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
}, {
|
|
Key: &resourcepb.ResourceKey{Namespace: ns2, Group: "folders", Resource: "folder", Name: "test2"},
|
|
Action: resourcepb.BulkRequest_ADDED,
|
|
Value: nil,
|
|
}}
|
|
|
|
resp2 := bulk.ProcessBulk(ctx, resource.BulkSettings{
|
|
Collection: collections2,
|
|
RebuildCollection: false,
|
|
}, toBulkIterator(bulkRequests2))
|
|
require.Nil(t, resp2.Error)
|
|
|
|
secondImport := time.Now()
|
|
|
|
// Verify that last imported times are combination of both bulk imports
|
|
result2 := collectLastImportedTimes(t, backend, ctx)
|
|
|
|
require.WithinDuration(t, result2[resource.NamespacedResource{Namespace: ns1, Group: "dashboards", Resource: "dashboard"}], secondImport, delta)
|
|
require.WithinDuration(t, result2[resource.NamespacedResource{Namespace: ns1, Group: "folders", Resource: "folder"}], firstImport, delta)
|
|
require.WithinDuration(t, result2[resource.NamespacedResource{Namespace: ns2, Group: "folders", Resource: "folder"}], secondImport, delta)
|
|
|
|
// Verify that last import time for ns1 folders are unchanged
|
|
ns1FoldersKey := resource.NamespacedResource{Namespace: ns1, Group: "folders", Resource: "folder"}
|
|
require.Equal(t, result1[ns1FoldersKey], result2[ns1FoldersKey])
|
|
})
|
|
}
|
|
|
|
func collectLastImportedTimes(t *testing.T, backend resource.StorageBackend, ctx context.Context) map[resource.NamespacedResource]time.Time {
|
|
result := map[resource.NamespacedResource]time.Time{}
|
|
for lm, err := range backend.GetResourceLastImportTimes(ctx) {
|
|
require.NoError(t, err)
|
|
result[lm.NamespacedResource] = lm.LastImportTime
|
|
}
|
|
return result
|
|
}
|
|
|
|
func toBulkIterator(reqs []*resourcepb.BulkRequest) resource.BulkRequestIterator {
|
|
it := &sliceBulkRequestIterator{}
|
|
*it = reqs
|
|
return it
|
|
}
|
|
|
|
type sliceBulkRequestIterator []*resourcepb.BulkRequest
|
|
|
|
func (s *sliceBulkRequestIterator) Next() bool {
|
|
if len(*s) > 1 {
|
|
*s = (*s)[1:]
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (s *sliceBulkRequestIterator) Request() *resourcepb.BulkRequest {
|
|
return (*s)[0]
|
|
}
|
|
|
|
func (s *sliceBulkRequestIterator) RollbackRequested() bool {
|
|
return false
|
|
}
|
|
|
|
func runTestIntegrationBackendOptimisticLocking(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
|
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
|
ns := nsPrefix + "-optimistic-locking"
|
|
|
|
t.Run("concurrent updates with same RV - only one succeeds", func(t *testing.T) {
|
|
// Create initial resource with rv0 (no previous RV)
|
|
rv0, err := writeEvent(ctx, backend, "concurrent-item", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
|
|
require.NoError(t, err)
|
|
require.Greater(t, rv0, int64(0))
|
|
|
|
// Launch 10 concurrent updates, all using rv0 as the previous RV
|
|
const numConcurrent = 10
|
|
type result struct {
|
|
rv int64
|
|
err error
|
|
}
|
|
results := make(chan result, numConcurrent)
|
|
|
|
// Start all goroutines concurrently
|
|
var wg sync.WaitGroup
|
|
wg.Add(numConcurrent)
|
|
for i := 0; i < numConcurrent; i++ {
|
|
go func(updateNum int) {
|
|
defer wg.Done()
|
|
rv, err := writeEvent(ctx, backend, "concurrent-item", resourcepb.WatchEvent_MODIFIED,
|
|
WithNamespaceAndRV(ns, rv0),
|
|
WithValue(fmt.Sprintf("update-%d", updateNum)))
|
|
results <- result{rv: rv, err: err}
|
|
}(i)
|
|
}
|
|
|
|
// Wait for all goroutines to complete
|
|
wg.Wait()
|
|
close(results)
|
|
|
|
// Count successes and failures
|
|
var successes, failures int
|
|
var successRV int64
|
|
for res := range results {
|
|
if res.err == nil {
|
|
successes++
|
|
successRV = res.rv
|
|
require.Greater(t, res.rv, rv0, "successful update should have higher RV than rv0")
|
|
} else {
|
|
failures++
|
|
}
|
|
}
|
|
|
|
// TODO: This test uses relaxed assertions instead of strict equality checks due to
|
|
// batch processing behavior in the SQL backend. When multiple concurrent updates
|
|
// with the same PreviousRV are batched together in a single transaction, only the
|
|
// first update in the batch can match the WHERE clause (resource_version = PreviousRV).
|
|
// Subsequent updates in the same batch fail to match (0 rows affected), causing
|
|
// checkConflict() to return an error, which rolls back the entire transaction.
|
|
// This results in all operations failing instead of the expected 1 success + 9 failures.
|
|
//
|
|
// Ideally, the ResourceVersionManager should either:
|
|
// 1. Detect conflicting PreviousRV values and prevent batching them together, OR
|
|
// 2. Handle the first operation's success separately before attempting remaining operations
|
|
//
|
|
// Until fixed, we verify "at most one success" instead of "exactly one success".
|
|
|
|
require.LessOrEqual(t, successes, 1, "at most one update should succeed")
|
|
require.GreaterOrEqual(t, failures, numConcurrent-1, "most concurrent updates should fail")
|
|
|
|
if successes == 1 {
|
|
// Verify the resource has the successful update
|
|
resp := backend.ReadResource(ctx, &resourcepb.ReadRequest{
|
|
Key: &resourcepb.ResourceKey{
|
|
Name: "concurrent-item",
|
|
Namespace: ns,
|
|
Group: "group",
|
|
Resource: "resource",
|
|
},
|
|
})
|
|
require.Nil(t, resp.Error)
|
|
require.Equal(t, successRV, resp.ResourceVersion, "resource should have the RV from the successful update")
|
|
}
|
|
})
|
|
|
|
t.Run("concurrent creates - only one succeeds", func(t *testing.T) {
|
|
// Launch 10 concurrent creates for the same resource name
|
|
const numConcurrent = 10
|
|
type result struct {
|
|
rv int64
|
|
err error
|
|
}
|
|
results := make([]result, numConcurrent)
|
|
|
|
// Start all goroutines concurrently
|
|
var wg sync.WaitGroup
|
|
wg.Add(numConcurrent)
|
|
for i := 0; i < numConcurrent; i++ {
|
|
go func(createNum int) {
|
|
defer wg.Done()
|
|
rv, err := writeEvent(ctx, backend, "concurrent-create-item", resourcepb.WatchEvent_ADDED,
|
|
WithNamespace(ns),
|
|
WithValue(fmt.Sprintf("create-%d", createNum)))
|
|
results[i] = result{rv: rv, err: err}
|
|
}(i)
|
|
}
|
|
|
|
// Wait for all goroutines to complete
|
|
wg.Wait()
|
|
|
|
// Count successes and failures
|
|
var successes int
|
|
var errorMessages []string
|
|
for _, res := range results {
|
|
if res.err == nil {
|
|
successes++
|
|
require.Greater(t, res.rv, int64(0), "successful create should have positive RV")
|
|
}
|
|
}
|
|
|
|
// Verify that exactly one create succeeded
|
|
// Note: Due to timing, it's possible that all creates detect each other and all fail.
|
|
// The important thing is that at most one succeeds (race condition is prevented).
|
|
require.LessOrEqual(t, successes, 1, "at most one create should succeed (errors: %v)", errorMessages)
|
|
})
|
|
}
|
|
|
|
func runTestIntegrationBackendKeyPathGeneration(t *testing.T, backend resource.StorageBackend, nsPrefix string, db sqldb.DB) {
|
|
ctx := testutil.NewDefaultTestContext(t)
|
|
|
|
t.Run("Create resource", func(t *testing.T) {
|
|
// Create a test resource
|
|
key := &resourcepb.ResourceKey{
|
|
Group: "playlist.grafana.app",
|
|
Resource: "playlists",
|
|
Namespace: nsPrefix + "-default",
|
|
Name: "test-playlist-crud",
|
|
}
|
|
|
|
// Create the K8s unstructured object
|
|
testObj := &unstructured.Unstructured{
|
|
Object: map[string]interface{}{
|
|
"apiVersion": "playlist.grafana.app/v0alpha1",
|
|
"kind": "Playlist",
|
|
"metadata": map[string]interface{}{
|
|
"name": "test-playlist-crud",
|
|
"namespace": nsPrefix + "-default",
|
|
"uid": "test-uid-crud-123",
|
|
},
|
|
"spec": map[string]interface{}{
|
|
"title": "My Test Playlist",
|
|
},
|
|
},
|
|
}
|
|
|
|
// Get metadata accessor
|
|
metaAccessor, err := utils.MetaAccessor(testObj)
|
|
require.NoError(t, err)
|
|
|
|
// Serialize to JSON
|
|
jsonBytes, err := testObj.MarshalJSON()
|
|
require.NoError(t, err)
|
|
|
|
// Create WriteEvent
|
|
writeEvent := resource.WriteEvent{
|
|
Type: resourcepb.WatchEvent_ADDED,
|
|
Key: key,
|
|
Value: jsonBytes,
|
|
Object: metaAccessor,
|
|
PreviousRV: 0, // Always 0 for new resources
|
|
GUID: "create-guid-crud-123",
|
|
}
|
|
|
|
// Create the resource using WriteEvent
|
|
createRV, err := backend.WriteEvent(ctx, writeEvent)
|
|
require.NoError(t, err)
|
|
require.Greater(t, createRV, int64(0))
|
|
|
|
// Verify created resource key_path
|
|
verifyKeyPath(t, db, ctx, key, "created", createRV, "")
|
|
|
|
t.Run("Update resource", func(t *testing.T) {
|
|
// Update the resource
|
|
testObj.Object["spec"] = map[string]interface{}{
|
|
"title": "My Updated Playlist",
|
|
}
|
|
|
|
updatedMetaAccessor, err := utils.MetaAccessor(testObj)
|
|
require.NoError(t, err)
|
|
|
|
updatedJsonBytes, err := testObj.MarshalJSON()
|
|
require.NoError(t, err)
|
|
|
|
updateEvent := resource.WriteEvent{
|
|
Type: resourcepb.WatchEvent_MODIFIED,
|
|
Key: key,
|
|
Value: updatedJsonBytes,
|
|
Object: updatedMetaAccessor,
|
|
PreviousRV: createRV,
|
|
GUID: fmt.Sprintf("update-guid-%d", createRV),
|
|
}
|
|
|
|
// Update the resource
|
|
updateRV, err := backend.WriteEvent(ctx, updateEvent)
|
|
require.NoError(t, err)
|
|
require.Greater(t, updateRV, createRV)
|
|
|
|
// Verify updated resource key_path
|
|
verifyKeyPath(t, db, ctx, key, "updated", updateRV, "")
|
|
|
|
t.Run("Delete resource", func(t *testing.T) {
|
|
deleteEvent := resource.WriteEvent{
|
|
Type: resourcepb.WatchEvent_DELETED,
|
|
Key: key,
|
|
Value: updatedJsonBytes, // Keep the last known value
|
|
Object: updatedMetaAccessor,
|
|
PreviousRV: updateRV,
|
|
GUID: fmt.Sprintf("delete-guid-%d", updateRV),
|
|
}
|
|
|
|
// Delete the resource
|
|
deleteRV, err := backend.WriteEvent(ctx, deleteEvent)
|
|
require.NoError(t, err)
|
|
require.Greater(t, deleteRV, updateRV)
|
|
|
|
// Verify deleted resource key_path
|
|
verifyKeyPath(t, db, ctx, key, "deleted", deleteRV, "")
|
|
})
|
|
})
|
|
})
|
|
|
|
t.Run("Resource with folder", func(t *testing.T) {
|
|
// Create a resource in a folder
|
|
folderKey := &resourcepb.ResourceKey{
|
|
Group: "dashboard.grafana.app",
|
|
Resource: "dashboards",
|
|
Namespace: nsPrefix + "-default",
|
|
Name: "my-dashboard",
|
|
}
|
|
|
|
// Create dashboard object with folder
|
|
dashboardObj := &unstructured.Unstructured{
|
|
Object: map[string]interface{}{
|
|
"apiVersion": "dashboard.grafana.app/v0alpha1",
|
|
"kind": "Dashboard",
|
|
"metadata": map[string]interface{}{
|
|
"name": "my-dashboard",
|
|
"namespace": nsPrefix + "-default",
|
|
"uid": "dash-uid-456",
|
|
"annotations": map[string]interface{}{
|
|
"grafana.app/folder": "test-folder",
|
|
},
|
|
},
|
|
"spec": map[string]interface{}{
|
|
"title": "My Dashboard",
|
|
},
|
|
},
|
|
}
|
|
|
|
folderMetaAccessor, err := utils.MetaAccessor(dashboardObj)
|
|
require.NoError(t, err)
|
|
|
|
folderJsonBytes, err := dashboardObj.MarshalJSON()
|
|
require.NoError(t, err)
|
|
|
|
folderWriteEvent := resource.WriteEvent{
|
|
Type: resourcepb.WatchEvent_ADDED,
|
|
Key: folderKey,
|
|
Value: folderJsonBytes,
|
|
Object: folderMetaAccessor,
|
|
PreviousRV: 0,
|
|
GUID: "folder-guid-456",
|
|
}
|
|
|
|
// Create the dashboard in folder
|
|
folderRV, err := backend.WriteEvent(ctx, folderWriteEvent)
|
|
require.NoError(t, err)
|
|
require.Greater(t, folderRV, int64(0))
|
|
|
|
// Verify folder resource key_path includes folder
|
|
verifyKeyPath(t, db, ctx, folderKey, "created", folderRV, "test-folder")
|
|
})
|
|
}
|
|
|
|
// verifyKeyPath is a helper function to verify key_path generation
|
|
func verifyKeyPath(t *testing.T, db sqldb.DB, ctx context.Context, key *resourcepb.ResourceKey, action string, resourceVersion int64, expectedFolder string) {
|
|
var query string
|
|
if db.DriverName() == "postgres" {
|
|
query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = $1 AND name = $2 AND resource_version = $3"
|
|
} else {
|
|
query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = ? AND name = ? AND resource_version = ?"
|
|
}
|
|
rows, err := db.QueryContext(ctx, query, key.Namespace, key.Name, resourceVersion)
|
|
require.NoError(t, err)
|
|
|
|
require.True(t, rows.Next())
|
|
|
|
var keyPath string
|
|
var actualRV int64
|
|
var actualAction int
|
|
var actualFolder string
|
|
|
|
err = rows.Scan(&keyPath, &actualRV, &actualAction, &actualFolder)
|
|
require.NoError(t, err)
|
|
err = rows.Close()
|
|
require.NoError(t, err)
|
|
|
|
// Verify basic key_path format
|
|
require.Contains(t, keyPath, "unified/data/")
|
|
require.Contains(t, keyPath, key.Group)
|
|
require.Contains(t, keyPath, key.Resource)
|
|
require.Contains(t, keyPath, key.Namespace)
|
|
require.Contains(t, keyPath, key.Name)
|
|
|
|
// Verify action suffix
|
|
require.Contains(t, keyPath, fmt.Sprintf("~%s~", action))
|
|
|
|
// Verify snowflake calculation
|
|
expectedSnowflake := (((resourceVersion / 1000) - snowflake.Epoch) << (snowflake.NodeBits + snowflake.StepBits)) + (resourceVersion % 1000)
|
|
require.Contains(t, keyPath, fmt.Sprintf("/%d~", expectedSnowflake), fmt.Sprintf("actual RV: %d", actualRV))
|
|
|
|
// Verify folder if specified
|
|
if expectedFolder != "" {
|
|
require.Equal(t, expectedFolder, actualFolder)
|
|
require.Contains(t, keyPath, expectedFolder)
|
|
}
|
|
|
|
// Verify action code matches
|
|
var expectedActionCode int
|
|
switch action {
|
|
case "created":
|
|
expectedActionCode = 1
|
|
case "updated":
|
|
expectedActionCode = 2
|
|
case "deleted":
|
|
expectedActionCode = 3
|
|
}
|
|
require.Equal(t, expectedActionCode, actualAction)
|
|
|
|
t.Logf("Action: %s, RV: %d, Snowflake: %d", action, resourceVersion, expectedSnowflake)
|
|
t.Logf("Key_path: %s", keyPath)
|
|
if expectedFolder != "" {
|
|
t.Logf("Folder: %s", actualFolder)
|
|
}
|
|
}
|