public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Storage: Return an iterator from backend List (#91185)

Browse Source
This commit is contained in:
Ryan McKinley
2024-07-31 12:05:59 +03:00
committed by GitHub
parent dd9172e738
commit f804b0baa3
13 changed files with 475 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/sql/test/integration_test.go
+61 -39
View File
@@ -26,7 +26,7 @@ func TestMain(m *testing.M) {
testsuite.Run(m)
}
func newServer(t *testing.T) sql.Backend {
func newServer(t *testing.T) (sql.Backend, resource.ResourceServer) {
t.Helper()
dbstore := infraDB.InitTestDB(t)
@@ -48,7 +48,15 @@ func newServer(t *testing.T) sql.Backend {
err = ret.Init(testutil.NewDefaultTestContext(t))
require.NoError(t, err)
return ret
server, err := resource.NewResourceServer(resource.ResourceServerOptions{
Backend: ret,
Diagnostics: ret,
Lifecycle: ret,
})
require.NoError(t, err)
require.NotNil(t, server)
return ret, server
}
func TestIntegrationBackendHappyPath(t *testing.T) {
@@ -57,57 +65,65 @@ func TestIntegrationBackendHappyPath(t *testing.T) {
t.Skip("skipping integration test")
}
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
store := newServer(t)
testUserA := &identity.StaticRequester{
Type: identity.TypeUser,
Login: "testuser",
UserID: 123,
UserUID: "u123",
OrgRole: identity.RoleAdmin,
IsGrafanaAdmin: true, // can do anything
}
ctx := identity.WithRequester(context.Background(), testUserA)
backend, server := newServer(t)
stream, err := store.WatchWriteEvents(ctx)
stream, err := backend.WatchWriteEvents(context.Background()) // Using a different context to avoid canceling the stream after the DefaultContextTimeout
require.NoError(t, err)
t.Run("Add 3 resources", func(t *testing.T) {
rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
rv, err := writeEvent(ctx, backend, "item1", resource.WatchEvent_ADDED)
require.NoError(t, err)
require.Equal(t, int64(1), rv)
rv, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
rv, err = writeEvent(ctx, backend, "item2", resource.WatchEvent_ADDED)
require.NoError(t, err)
require.Equal(t, int64(2), rv)
rv, err = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED)
rv, err = writeEvent(ctx, backend, "item3", resource.WatchEvent_ADDED)
require.NoError(t, err)
require.Equal(t, int64(3), rv)
})
t.Run("Update item2", func(t *testing.T) {
rv, err := writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED)
rv, err := writeEvent(ctx, backend, "item2", resource.WatchEvent_MODIFIED)
require.NoError(t, err)
require.Equal(t, int64(4), rv)
})
t.Run("Delete item1", func(t *testing.T) {
rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_DELETED)
rv, err := writeEvent(ctx, backend, "item1", resource.WatchEvent_DELETED)
require.NoError(t, err)
require.Equal(t, int64(5), rv)
})
t.Run("Read latest item 2", func(t *testing.T) {
resp := store.ReadResource(ctx, &resource.ReadRequest{Key: resourceKey("item2")})
require.NoError(t, err)
resp := backend.ReadResource(ctx, &resource.ReadRequest{Key: resourceKey("item2")})
require.Nil(t, resp.Error)
require.Equal(t, int64(4), resp.ResourceVersion)
require.Equal(t, "item2 MODIFIED", string(resp.Value))
})
t.Run("Read early verion of item2", func(t *testing.T) {
resp := store.ReadResource(ctx, &resource.ReadRequest{
t.Run("Read early version of item2", func(t *testing.T) {
resp := backend.ReadResource(ctx, &resource.ReadRequest{
Key: resourceKey("item2"),
ResourceVersion: 3, // item2 was created at rv=2 and updated at rv=4
})
require.NoError(t, err)
require.Nil(t, resp.Error)
require.Equal(t, int64(2), resp.ResourceVersion)
require.Equal(t, "item2 ADDED", string(resp.Value))
})
t.Run("PrepareList latest", func(t *testing.T) {
resp := store.PrepareList(ctx, &resource.ListRequest{
resp, err := server.List(ctx, &resource.ListRequest{
Options: &resource.ListOptions{
Key: &resource.ResourceKey{
Namespace: "namespace",
@@ -117,6 +133,7 @@ func TestIntegrationBackendHappyPath(t *testing.T) {
},
})
require.NoError(t, err)
require.Nil(t, resp.Error)
require.Len(t, resp.Items, 2)
require.Equal(t, "item2 MODIFIED", string(resp.Items[0].Value))
require.Equal(t, "item3 ADDED", string(resp.Items[1].Value))
@@ -157,42 +174,42 @@ func TestIntegrationBackendWatchWriteEventsFromLastest(t *testing.T) {
}
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
store := newServer(t)
backend, _ := newServer(t)
// Create a few resources before initing the watch
_, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
_, err := writeEvent(ctx, backend, "item1", resource.WatchEvent_ADDED)
require.NoError(t, err)
// Start the watch
stream, err := store.WatchWriteEvents(ctx)
stream, err := backend.WatchWriteEvents(ctx)
require.NoError(t, err)
// Create one more event
_, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
_, err = writeEvent(ctx, backend, "item2", resource.WatchEvent_ADDED)
require.NoError(t, err)
require.Equal(t, "item2", (<-stream).Key.Name)
}
func TestIntegrationBackendPrepareList(t *testing.T) {
func TestIntegrationBackendList(t *testing.T) {
t.Skip("TODO: test blocking, skipping to unblock Enterprise until we fix this")
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
store := newServer(t)
backend, server := newServer(t)
// Create a few resources before initing the watch
_, _ = writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED) // rv=1
_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED) // rv=2 - will be modified at rv=6
_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED) // rv=3 - will be deleted at rv=7
_, _ = writeEvent(ctx, store, "item4", resource.WatchEvent_ADDED) // rv=4
_, _ = writeEvent(ctx, store, "item5", resource.WatchEvent_ADDED) // rv=5
_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED) // rv=6
_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_DELETED) // rv=7
_, _ = writeEvent(ctx, store, "item6", resource.WatchEvent_ADDED) // rv=8
// Create a few resources before starting the watch
_, _ = writeEvent(ctx, backend, "item1", resource.WatchEvent_ADDED) // rv=1
_, _ = writeEvent(ctx, backend, "item2", resource.WatchEvent_ADDED) // rv=2 - will be modified at rv=6
_, _ = writeEvent(ctx, backend, "item3", resource.WatchEvent_ADDED) // rv=3 - will be deleted at rv=7
_, _ = writeEvent(ctx, backend, "item4", resource.WatchEvent_ADDED) // rv=4
_, _ = writeEvent(ctx, backend, "item5", resource.WatchEvent_ADDED) // rv=5
_, _ = writeEvent(ctx, backend, "item2", resource.WatchEvent_MODIFIED) // rv=6
_, _ = writeEvent(ctx, backend, "item3", resource.WatchEvent_DELETED) // rv=7
_, _ = writeEvent(ctx, backend, "item6", resource.WatchEvent_ADDED) // rv=8
t.Run("fetch all latest", func(t *testing.T) {
res := store.PrepareList(ctx, &resource.ListRequest{
res, err := server.List(ctx, &resource.ListRequest{
Options: &resource.ListOptions{
Key: &resource.ResourceKey{
Group: "group",
@@ -200,6 +217,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
},
},
})
require.NoError(t, err)
require.Nil(t, res.Error)
require.Len(t, res.Items, 5)
// should be sorted by resource version DESC
@@ -213,7 +231,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
})
t.Run("list latest first page ", func(t *testing.T) {
res := store.PrepareList(ctx, &resource.ListRequest{
res, err := server.List(ctx, &resource.ListRequest{
Limit: 3,
Options: &resource.ListOptions{
Key: &resource.ResourceKey{
@@ -222,6 +240,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
},
},
})
require.NoError(t, err)
require.Nil(t, res.Error)
require.Len(t, res.Items, 3)
continueToken, err := sql.GetContinueToken(res.NextPageToken)
@@ -230,11 +249,10 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
require.Equal(t, "item2 MODIFIED", string(res.Items[1].Value))
require.Equal(t, "item5 ADDED", string(res.Items[2].Value))
require.Equal(t, int64(8), continueToken.ResourceVersion)
require.Equal(t, int64(3), continueToken.StartOffset)
})
t.Run("list at revision", func(t *testing.T) {
res := store.PrepareList(ctx, &resource.ListRequest{
res, err := server.List(ctx, &resource.ListRequest{
ResourceVersion: 4,
Options: &resource.ListOptions{
Key: &resource.ResourceKey{
@@ -243,6 +261,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
},
},
})
require.NoError(t, err)
require.Nil(t, res.Error)
require.Len(t, res.Items, 4)
require.Equal(t, "item4 ADDED", string(res.Items[0].Value))
@@ -253,7 +272,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
})
t.Run("fetch first page at revision with limit", func(t *testing.T) {
res := store.PrepareList(ctx, &resource.ListRequest{
res, err := server.List(ctx, &resource.ListRequest{
Limit: 3,
ResourceVersion: 7,
Options: &resource.ListOptions{
@@ -263,6 +282,8 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
},
},
})
require.NoError(t, err)
require.NoError(t, err)
require.Nil(t, res.Error)
require.Len(t, res.Items, 3)
t.Log(res.Items)
@@ -273,7 +294,6 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
continueToken, err := sql.GetContinueToken(res.NextPageToken)
require.NoError(t, err)
require.Equal(t, int64(7), continueToken.ResourceVersion)
require.Equal(t, int64(3), continueToken.StartOffset)
})
t.Run("fetch second page at revision", func(t *testing.T) {
@@ -281,7 +301,7 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
ResourceVersion: 8,
StartOffset: 2,
}
res := store.PrepareList(ctx, &resource.ListRequest{
res, err := server.List(ctx, &resource.ListRequest{
NextPageToken: continueToken.String(),
Limit: 2,
Options: &resource.ListOptions{
@@ -291,12 +311,14 @@ func TestIntegrationBackendPrepareList(t *testing.T) {
},
},
})
require.NoError(t, err)
require.Nil(t, res.Error)
require.Len(t, res.Items, 2)
t.Log(res.Items)
require.Equal(t, "item5 ADDED", string(res.Items[0].Value))
require.Equal(t, "item4 ADDED", string(res.Items[1].Value))
continueToken, err := sql.GetContinueToken(res.NextPageToken)
continueToken, err = sql.GetContinueToken(res.NextPageToken)
require.NoError(t, err)
require.Equal(t, int64(8), continueToken.ResourceVersion)
require.Equal(t, int64(4), continueToken.StartOffset)