LastImportTime for resource. (#112153)
* LastImportTime for resource. * Make StorageBackendImpl implement GetResourceLastImportTimes * More missing implementations of GetResourceLastImportTimes * Fix import. * Skip TestGetResourceLastImportTime in TestBadgerKVStorageBackend. * Implement GetResourceLastImportTimes by mockStorageBackend * Bump test tolerance. * Fix postgres query and timezone. * Fix postgres query and timezone. * Make linter happy.
This commit is contained in:
@@ -242,6 +242,12 @@ func (a *dashboardSqlAccess) ListModifiedSince(ctx context.Context, key resource
|
||||
}
|
||||
}
|
||||
|
||||
func (a *dashboardSqlAccess) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
|
||||
return func(yield func(resource.ResourceLastImportTime, error) bool) {
|
||||
yield(resource.ResourceLastImportTime{}, errors.New("not implemented"))
|
||||
}
|
||||
}
|
||||
|
||||
// List implements StorageBackend.
|
||||
func (a *dashboardSqlAccess) ListIterator(ctx context.Context, req *resourcepb.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
|
||||
if req.ResourceVersion != 0 {
|
||||
|
||||
@@ -61,3 +61,9 @@ func (c *StorageBackendImpl) WatchWriteEvents(ctx context.Context) (<-chan *reso
|
||||
func (c *StorageBackendImpl) WriteEvent(context.Context, resource.WriteEvent) (int64, error) {
|
||||
return 0, errNoopStorage
|
||||
}
|
||||
|
||||
func (c *StorageBackendImpl) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
|
||||
return func(yield func(resource.ResourceLastImportTime, error) bool) {
|
||||
yield(resource.ResourceLastImportTime{}, errNoopStorage)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/endpoints/request"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
|
||||
"github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/iam/common"
|
||||
@@ -295,3 +296,9 @@ func (s *ResourcePermSqlBackend) WriteEvent(ctx context.Context, event resource.
|
||||
|
||||
return rv, err
|
||||
}
|
||||
|
||||
func (s *ResourcePermSqlBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
|
||||
return func(yield func(resource.ResourceLastImportTime, error) bool) {
|
||||
yield(resource.ResourceLastImportTime{}, errNotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,12 +33,13 @@ func grpcMetaValueIsTrue(vals []string) bool {
|
||||
}
|
||||
|
||||
type BulkRequestIterator interface {
|
||||
// Next advances the iterator to the next element if one exists.
|
||||
Next() bool
|
||||
|
||||
// The next event we should process
|
||||
// Request returns the current element. Only valid after Next() returns true.
|
||||
Request() *resourcepb.BulkRequest
|
||||
|
||||
// Rollback requested
|
||||
// RollbackRequested returns true if there was an error advancing the iterator. Checked after Next() returns true.
|
||||
RollbackRequested() bool
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,12 @@ type cdkBackend struct {
|
||||
stream chan<- *WrittenEvent
|
||||
}
|
||||
|
||||
func (s *cdkBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {
|
||||
return func(yield func(ResourceLastImportTime, error) bool) {
|
||||
yield(ResourceLastImportTime{}, errors.New("not implemented"))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *cdkBackend) ListModifiedSince(ctx context.Context, key NamespacedResource, sinceRv int64) (int64, iter.Seq2[*ModifiedResource, error]) {
|
||||
return 0, func(yield func(*ModifiedResource, error) bool) {
|
||||
yield(nil, errors.New("not implemented"))
|
||||
|
||||
@@ -129,6 +129,12 @@ func (m *mockStorageBackend) ListModifiedSince(ctx context.Context, key Namespac
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockStorageBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {
|
||||
return func(yield func(ResourceLastImportTime, error) bool) {
|
||||
yield(ResourceLastImportTime{}, errors.New("not implemented"))
|
||||
}
|
||||
}
|
||||
|
||||
// mockSearchBackend implements SearchBackend for testing with tracking capabilities
|
||||
type mockSearchBackend struct {
|
||||
openIndexes []NamespacedResource
|
||||
|
||||
@@ -86,6 +86,11 @@ type BackendReadResponse struct {
|
||||
Error *resourcepb.ErrorResult
|
||||
}
|
||||
|
||||
type ResourceLastImportTime struct {
|
||||
NamespacedResource
|
||||
LastImportTime time.Time
|
||||
}
|
||||
|
||||
// The StorageBackend is an internal abstraction that supports interacting with
|
||||
// the underlying raw storage medium. This interface is never exposed directly,
|
||||
// it is provided by concrete instances that actually write values.
|
||||
@@ -118,6 +123,9 @@ type StorageBackend interface {
|
||||
|
||||
// Get resource stats within the storage backend. When namespace is empty, it will apply to all
|
||||
GetResourceStats(ctx context.Context, namespace string, minCount int) ([]ResourceStats, error)
|
||||
|
||||
// GetResourceLastImportTimes returns import times for all namespaced resources in the backend.
|
||||
GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error]
|
||||
}
|
||||
|
||||
type ModifiedResource struct {
|
||||
|
||||
@@ -15,12 +15,13 @@ import (
|
||||
|
||||
"github.com/bwmarrin/snowflake"
|
||||
"github.com/grafana/grafana-app-sdk/logging"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/debouncer"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/debouncer"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -1076,6 +1077,12 @@ func (k *kvStorageBackend) GetResourceStats(ctx context.Context, namespace strin
|
||||
return k.dataStore.GetResourceStats(ctx, namespace, minCount)
|
||||
}
|
||||
|
||||
func (k *kvStorageBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {
|
||||
return func(yield func(ResourceLastImportTime, error) bool) {
|
||||
yield(ResourceLastImportTime{}, fmt.Errorf("not implemented"))
|
||||
}
|
||||
}
|
||||
|
||||
// readAndClose reads all data from a ReadCloser and ensures it's closed,
|
||||
// combining any errors from both operations.
|
||||
func readAndClose(r io.ReadCloser) ([]byte, error) {
|
||||
|
||||
@@ -1489,8 +1489,8 @@ func TestConcurrentIndexUpdateAndSearchWithIndexMinUpdateInterval(t *testing.T)
|
||||
if rvDiff == 0 {
|
||||
// OK
|
||||
} else {
|
||||
// Allow returned RV to be within 10% of minInterval.
|
||||
require.InDelta(t, minInterval.Milliseconds(), rvDiff, float64(minInterval.Milliseconds())*0.10)
|
||||
// Allow returned RV to be within 20% of minInterval (to account for slow CI machines).
|
||||
require.InDelta(t, minInterval.Milliseconds(), rvDiff, float64(minInterval.Milliseconds())*0.20)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -955,3 +955,51 @@ func (b *backend) fetchLatestHistoryRV(ctx context.Context, x db.ContextExecer,
|
||||
}
|
||||
return res.ResourceVersion, nil
|
||||
}
|
||||
|
||||
func (b *backend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
|
||||
ctx, span := b.tracer.Start(ctx, tracePrefix+"GetLastImportTimes")
|
||||
defer span.End()
|
||||
|
||||
rows, err := dbutil.QueryRows(ctx, b.db, sqlResourceLastImportTimeQuery, &sqlResourceLastImportTimeQueryRequest{SQLTemplate: sqltemplate.New(b.dialect)})
|
||||
if err != nil {
|
||||
return func(yield func(resource.ResourceLastImportTime, error) bool) {
|
||||
yield(resource.ResourceLastImportTime{}, err)
|
||||
}
|
||||
}
|
||||
|
||||
return func(yield func(resource.ResourceLastImportTime, error) bool) {
|
||||
closeOnDefer := true
|
||||
defer func() {
|
||||
if closeOnDefer {
|
||||
_ = rows.Close() // Close while ignoring errors.
|
||||
}
|
||||
}()
|
||||
|
||||
for rows.Next() {
|
||||
// If context has finished, return early.
|
||||
if ctx.Err() != nil {
|
||||
yield(resource.ResourceLastImportTime{}, ctx.Err())
|
||||
return
|
||||
}
|
||||
|
||||
row := resource.ResourceLastImportTime{}
|
||||
err = rows.Scan(&row.Namespace, &row.Group, &row.Resource, &row.LastImportTime)
|
||||
if err != nil {
|
||||
yield(resource.ResourceLastImportTime{}, err)
|
||||
return
|
||||
}
|
||||
|
||||
if !yield(row, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
closeOnDefer = false
|
||||
|
||||
// Close and report error, if any.
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
yield(resource.ResourceLastImportTime{}, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
|
||||
// Calculate the RV based on incoming request timestamps
|
||||
rv := newBulkRV()
|
||||
|
||||
summaries := make(map[string]*resourcepb.BulkResponse_Summary, len(setting.Collection)*4)
|
||||
summaries := make(map[string]*resourcepb.BulkResponse_Summary, len(setting.Collection))
|
||||
|
||||
// First clear everything in the transaction
|
||||
if setting.RebuildCollection {
|
||||
@@ -182,6 +182,14 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
|
||||
summaries[resource.NSGR(key)] = summary
|
||||
rsp.Summary = append(rsp.Summary, summary)
|
||||
}
|
||||
} else {
|
||||
for _, key := range setting.Collection {
|
||||
summaries[resource.NSGR(key)] = &resourcepb.BulkResponse_Summary{
|
||||
Namespace: key.Namespace,
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
obj := &unstructured.Unstructured{}
|
||||
@@ -253,6 +261,12 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
|
||||
if err != nil {
|
||||
b.log.Warn("error increasing RV", "error", err)
|
||||
}
|
||||
|
||||
// Update the last import time. This is important to trigger reindexing
|
||||
// of the resource for a given namespace.
|
||||
if err := b.updateLastImportTime(ctx, tx, key, time.Now()); err != nil {
|
||||
return rollbackWithError(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -262,6 +276,19 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
|
||||
return rsp
|
||||
}
|
||||
|
||||
func (b *backend) updateLastImportTime(ctx context.Context, tx db.Tx, key *resourcepb.ResourceKey, now time.Time) error {
|
||||
if _, err := dbutil.Exec(ctx, tx, sqlResourceLastImportTimeInsert, sqlResourceLastImportTimeInsertRequest{
|
||||
SQLTemplate: sqltemplate.New(b.dialect),
|
||||
Namespace: key.Namespace,
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
LastImportTime: now.UTC(),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert resource last import time: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type bulkWroker struct {
|
||||
ctx context.Context
|
||||
tx db.ContextExecer
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{{ if eq $.DialectName "mysql" }}
|
||||
INSERT INTO {{ .Ident "resource_last_import_time" }} (
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "namespace" }},
|
||||
{{ .Ident "last_import_time" }}
|
||||
) VALUES (
|
||||
{{ .Arg .Group }},
|
||||
{{ .Arg .Resource }},
|
||||
{{ .Arg .Namespace }},
|
||||
{{ .Arg .LastImportTime }}
|
||||
) ON DUPLICATE KEY UPDATE {{ .Ident "last_import_time" }} = {{ .Arg .LastImportTime }}
|
||||
|
||||
{{ else if eq $.DialectName "sqlite" }}
|
||||
INSERT OR REPLACE INTO {{ .Ident "resource_last_import_time" }} (
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "namespace" }},
|
||||
{{ .Ident "last_import_time" }}
|
||||
) VALUES (
|
||||
{{ .Arg .Group }},
|
||||
{{ .Arg .Resource }},
|
||||
{{ .Arg .Namespace }},
|
||||
{{ .Arg .LastImportTime }}
|
||||
)
|
||||
|
||||
{{ else if eq $.DialectName "postgres" }}
|
||||
INSERT INTO {{ .Ident "resource_last_import_time" }} (
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "namespace" }},
|
||||
{{ .Ident "last_import_time" }}
|
||||
) VALUES (
|
||||
{{ .Arg .Group }},
|
||||
{{ .Arg .Resource }},
|
||||
{{ .Arg .Namespace }},
|
||||
{{ .Arg .LastImportTime }}
|
||||
) ON CONFLICT ({{ .Ident "group" }}, {{ .Ident "resource" }}, {{ .Ident "namespace" }})
|
||||
DO UPDATE SET {{ .Ident "last_import_time" }} = {{ .Arg .LastImportTime }}
|
||||
|
||||
{{ end }}
|
||||
;
|
||||
@@ -0,0 +1,8 @@
|
||||
SELECT
|
||||
{{ .Ident "namespace" }},
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "last_import_time" }}
|
||||
FROM
|
||||
{{ .Ident "resource_last_import_time" }}
|
||||
;
|
||||
@@ -116,6 +116,17 @@ func initResourceTables(mg *migrator.Migrator) string {
|
||||
},
|
||||
})
|
||||
|
||||
tables = append(tables, migrator.Table{
|
||||
Name: "resource_last_import_time",
|
||||
Columns: []*migrator.Column{
|
||||
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
|
||||
{Name: "last_import_time", Type: migrator.DB_DateTime, Nullable: false},
|
||||
},
|
||||
PrimaryKeys: []string{"group", "resource", "namespace"},
|
||||
})
|
||||
|
||||
// Initialize all tables
|
||||
for t := range tables {
|
||||
mg.AddMigration("drop table "+tables[t].Name, migrator.NewDropTableMigration(tables[t].Name))
|
||||
|
||||
@@ -58,6 +58,9 @@ var (
|
||||
|
||||
sqlResourceBlobInsert = mustTemplate("resource_blob_insert.sql")
|
||||
sqlResourceBlobQuery = mustTemplate("resource_blob_query.sql")
|
||||
|
||||
sqlResourceLastImportTimeInsert = mustTemplate("resource_last_import_time_insert.sql")
|
||||
sqlResourceLastImportTimeQuery = mustTemplate("resource_last_import_time_query.sql")
|
||||
)
|
||||
|
||||
// TxOptions.
|
||||
@@ -454,3 +457,35 @@ func (r sqlResourceListModifiedSinceRequest) Validate() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type sqlResourceLastImportTimeInsertRequest struct {
|
||||
sqltemplate.SQLTemplate
|
||||
Namespace string
|
||||
Group string
|
||||
Resource string
|
||||
LastImportTime time.Time
|
||||
}
|
||||
|
||||
func (r sqlResourceLastImportTimeInsertRequest) Validate() error {
|
||||
if r.Namespace == "" {
|
||||
return fmt.Errorf("missing namespace")
|
||||
}
|
||||
if r.Group == "" {
|
||||
return fmt.Errorf("missing group")
|
||||
}
|
||||
if r.Resource == "" {
|
||||
return fmt.Errorf("missing resource")
|
||||
}
|
||||
if r.LastImportTime.IsZero() {
|
||||
return fmt.Errorf("last import time cannot be zero")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type sqlResourceLastImportTimeQueryRequest struct {
|
||||
sqltemplate.SQLTemplate
|
||||
}
|
||||
|
||||
func (r *sqlResourceLastImportTimeQueryRequest) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -495,5 +495,25 @@ func TestUnifiedStorageQueries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceLastImportTimeInsert: {
|
||||
{
|
||||
Name: "insert",
|
||||
Data: &sqlResourceLastImportTimeInsertRequest{
|
||||
SQLTemplate: mocks.NewTestingSQLTemplate(),
|
||||
Namespace: "ns",
|
||||
Group: "group",
|
||||
Resource: "res",
|
||||
LastImportTime: time.Date(2025, 10, 07, 22, 30, 05, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceLastImportTimeQuery: {
|
||||
{
|
||||
Name: "insert",
|
||||
Data: &sqlResourceLastImportTimeQueryRequest{
|
||||
SQLTemplate: mocks.NewTestingSQLTemplate(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}})
|
||||
}
|
||||
|
||||
Vendored
Executable
+12
@@ -0,0 +1,12 @@
|
||||
INSERT INTO `resource_last_import_time` (
|
||||
`group`,
|
||||
`resource`,
|
||||
`namespace`,
|
||||
`last_import_time`
|
||||
) VALUES (
|
||||
'group',
|
||||
'res',
|
||||
'ns',
|
||||
'2025-10-07 22:30:05 +0000 UTC'
|
||||
) ON DUPLICATE KEY UPDATE `last_import_time` = '2025-10-07 22:30:05 +0000 UTC'
|
||||
;
|
||||
Vendored
Executable
+8
@@ -0,0 +1,8 @@
|
||||
SELECT
|
||||
`namespace`,
|
||||
`group`,
|
||||
`resource`,
|
||||
`last_import_time`
|
||||
FROM
|
||||
`resource_last_import_time`
|
||||
;
|
||||
Vendored
Executable
+13
@@ -0,0 +1,13 @@
|
||||
INSERT INTO "resource_last_import_time" (
|
||||
"group",
|
||||
"resource",
|
||||
"namespace",
|
||||
"last_import_time"
|
||||
) VALUES (
|
||||
'group',
|
||||
'res',
|
||||
'ns',
|
||||
'2025-10-07 22:30:05 +0000 UTC'
|
||||
) ON CONFLICT ("group", "resource", "namespace")
|
||||
DO UPDATE SET "last_import_time" = '2025-10-07 22:30:05 +0000 UTC'
|
||||
;
|
||||
Vendored
Executable
+8
@@ -0,0 +1,8 @@
|
||||
SELECT
|
||||
"namespace",
|
||||
"group",
|
||||
"resource",
|
||||
"last_import_time"
|
||||
FROM
|
||||
"resource_last_import_time"
|
||||
;
|
||||
Vendored
Executable
+12
@@ -0,0 +1,12 @@
|
||||
INSERT OR REPLACE INTO "resource_last_import_time" (
|
||||
"group",
|
||||
"resource",
|
||||
"namespace",
|
||||
"last_import_time"
|
||||
) VALUES (
|
||||
'group',
|
||||
'res',
|
||||
'ns',
|
||||
'2025-10-07 22:30:05 +0000 UTC'
|
||||
)
|
||||
;
|
||||
Vendored
Executable
+8
@@ -0,0 +1,8 @@
|
||||
SELECT
|
||||
"namespace",
|
||||
"group",
|
||||
"resource",
|
||||
"last_import_time"
|
||||
FROM
|
||||
"resource_last_import_time"
|
||||
;
|
||||
@@ -39,6 +39,7 @@ const (
|
||||
TestListModifiedSince = "list events since rv"
|
||||
TestListTrash = "list trash"
|
||||
TestCreateNewResource = "create new resource"
|
||||
TestGetResourceLastImportTime = "get resource last import time"
|
||||
)
|
||||
|
||||
type NewBackendFunc func(ctx context.Context) resource.StorageBackend
|
||||
@@ -81,6 +82,7 @@ func RunStorageBackendTest(t *testing.T, newBackend NewBackendFunc, opts *TestOp
|
||||
{TestListTrash, runTestIntegrationBackendTrash},
|
||||
{TestCreateNewResource, runTestIntegrationBackendCreateNewResource},
|
||||
{TestListModifiedSince, runTestIntegrationBackendListModifiedSince},
|
||||
{TestGetResourceLastImportTime, runTestIntegrationGetResourceLastImportTime},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -1375,3 +1377,168 @@ func runTestIntegrationBackendTrash(t *testing.T, backend resource.StorageBacken
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runTestIntegrationGetResourceLastImportTime(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
|
||||
bulk, ok := backend.(resource.BulkProcessingBackend)
|
||||
if !ok {
|
||||
// This test is only enabled for backends that DO support bulk import. If the backend does not support
|
||||
// bulk import but has this test enabled, that's a bug.
|
||||
t.Fatal("backend does not support bulk import")
|
||||
}
|
||||
|
||||
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
|
||||
|
||||
t.Run("no imported times by default", func(t *testing.T) {
|
||||
res := collectLastImportedTimes(t, backend, ctx)
|
||||
require.Empty(t, res)
|
||||
})
|
||||
|
||||
t.Run("last imported time after bulk import", func(t *testing.T) {
|
||||
ns := nsPrefix + "-import"
|
||||
|
||||
collections := []*resourcepb.ResourceKey{
|
||||
{Namespace: ns, Group: "dashboards", Resource: "dashboard"},
|
||||
{Namespace: ns, Group: "folders", Resource: "folder"},
|
||||
}
|
||||
|
||||
bulkRequests := []*resourcepb.BulkRequest{
|
||||
{
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns, Group: "dashboards", Resource: "dashboard", Name: "test"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
},
|
||||
{
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns, Group: "dashboards", Resource: "dashboard", Name: "test2"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
},
|
||||
{
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns, Group: "folders", Resource: "folder", Name: "test2"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
},
|
||||
}
|
||||
|
||||
resp := bulk.ProcessBulk(ctx, resource.BulkSettings{
|
||||
Collection: collections,
|
||||
RebuildCollection: true,
|
||||
}, toBulkIterator(bulkRequests))
|
||||
require.Nil(t, resp.Error)
|
||||
|
||||
result := collectLastImportedTimes(t, backend, ctx)
|
||||
require.Len(t, result, len(collections))
|
||||
|
||||
now := time.Now()
|
||||
|
||||
for _, r := range collections {
|
||||
nsr := resource.NamespacedResource{Namespace: r.Namespace, Group: r.Group, Resource: r.Resource}
|
||||
lastImported, ok := result[nsr]
|
||||
require.True(t, ok, "resource not found: %s", nsr.String())
|
||||
require.True(t, lastImported.After(now.Add(-5*time.Second)), "last imported time is not recent")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("last imported time after multiple bulk imports", func(t *testing.T) {
|
||||
ns1 := nsPrefix + "-import1"
|
||||
collections1 := []*resourcepb.ResourceKey{
|
||||
{Namespace: ns1, Group: "dashboards", Resource: "dashboard"},
|
||||
{Namespace: ns1, Group: "folders", Resource: "folder"},
|
||||
}
|
||||
bulkRequests1 := []*resourcepb.BulkRequest{{
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "dashboards", Resource: "dashboard", Name: "test"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
}, {
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "dashboards", Resource: "dashboard", Name: "test2"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
}, {
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "folders", Resource: "folder", Name: "test2"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
}}
|
||||
|
||||
resp1 := bulk.ProcessBulk(ctx, resource.BulkSettings{
|
||||
Collection: collections1,
|
||||
RebuildCollection: true,
|
||||
}, toBulkIterator(bulkRequests1))
|
||||
require.Nil(t, resp1.Error)
|
||||
|
||||
firstImport := time.Now()
|
||||
|
||||
const delta = 5 * time.Second
|
||||
// Verify that last imported times are combination of both bulk imports
|
||||
result1 := collectLastImportedTimes(t, backend, ctx)
|
||||
require.WithinDuration(t, result1[resource.NamespacedResource{Namespace: ns1, Group: "dashboards", Resource: "dashboard"}], firstImport, delta)
|
||||
require.WithinDuration(t, result1[resource.NamespacedResource{Namespace: ns1, Group: "folders", Resource: "folder"}], firstImport, delta)
|
||||
|
||||
// Do another bulk import, without overwriting existing resources. We import into ns1-dashboards (same as before),
|
||||
// and new ns2-folders. ns1-folders is unchanged.
|
||||
ns2 := nsPrefix + "-import2"
|
||||
collections2 := []*resourcepb.ResourceKey{
|
||||
{Namespace: ns1, Group: "dashboards", Resource: "dashboard"}, // Import MORE dashboards into ns1
|
||||
{Namespace: ns2, Group: "folders", Resource: "folder"},
|
||||
}
|
||||
bulkRequests2 := []*resourcepb.BulkRequest{{
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns1, Group: "dashboards", Resource: "dashboard", Name: "new-test"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
}, {
|
||||
Key: &resourcepb.ResourceKey{Namespace: ns2, Group: "folders", Resource: "folder", Name: "test2"},
|
||||
Action: resourcepb.BulkRequest_ADDED,
|
||||
Value: nil,
|
||||
}}
|
||||
|
||||
resp2 := bulk.ProcessBulk(ctx, resource.BulkSettings{
|
||||
Collection: collections2,
|
||||
RebuildCollection: false,
|
||||
}, toBulkIterator(bulkRequests2))
|
||||
require.Nil(t, resp2.Error)
|
||||
|
||||
secondImport := time.Now()
|
||||
|
||||
// Verify that last imported times are combination of both bulk imports
|
||||
result2 := collectLastImportedTimes(t, backend, ctx)
|
||||
|
||||
require.WithinDuration(t, result2[resource.NamespacedResource{Namespace: ns1, Group: "dashboards", Resource: "dashboard"}], secondImport, delta)
|
||||
require.WithinDuration(t, result2[resource.NamespacedResource{Namespace: ns1, Group: "folders", Resource: "folder"}], firstImport, delta)
|
||||
require.WithinDuration(t, result2[resource.NamespacedResource{Namespace: ns2, Group: "folders", Resource: "folder"}], secondImport, delta)
|
||||
|
||||
// Verify that last import time for ns1 folders are unchanged
|
||||
ns1FoldersKey := resource.NamespacedResource{Namespace: ns1, Group: "folders", Resource: "folder"}
|
||||
require.Equal(t, result1[ns1FoldersKey], result2[ns1FoldersKey])
|
||||
})
|
||||
}
|
||||
|
||||
func collectLastImportedTimes(t *testing.T, backend resource.StorageBackend, ctx context.Context) map[resource.NamespacedResource]time.Time {
|
||||
result := map[resource.NamespacedResource]time.Time{}
|
||||
for lm, err := range backend.GetResourceLastImportTimes(ctx) {
|
||||
require.NoError(t, err)
|
||||
result[lm.NamespacedResource] = lm.LastImportTime
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toBulkIterator(reqs []*resourcepb.BulkRequest) resource.BulkRequestIterator {
|
||||
it := &sliceBulkRequestIterator{}
|
||||
*it = reqs
|
||||
return it
|
||||
}
|
||||
|
||||
type sliceBulkRequestIterator []*resourcepb.BulkRequest
|
||||
|
||||
func (s *sliceBulkRequestIterator) Next() bool {
|
||||
if len(*s) > 1 {
|
||||
*s = (*s)[1:]
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *sliceBulkRequestIterator) Request() *resourcepb.BulkRequest {
|
||||
return (*s)[0]
|
||||
}
|
||||
|
||||
func (s *sliceBulkRequestIterator) RollbackRequested() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -30,6 +30,8 @@ func TestBadgerKVStorageBackend(t *testing.T) {
|
||||
// TODO: fix these tests and remove this skip
|
||||
TestBlobSupport: true,
|
||||
TestListModifiedSince: true,
|
||||
// Badger does not support bulk import yet.
|
||||
TestGetResourceLastImportTime: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user