From ecadd9945658a1ef6e2ad94a25d5ce5d5a8affb0 Mon Sep 17 00:00:00 2001 From: Arati R <33031346+suntala@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:17:27 +0200 Subject: [PATCH] SearchV2: Support soft deletion (#90217) * soft delete * Fix bench test Co-authored-by: Bruno Abrantes * Add integration test for soft deletion --------- Co-authored-by: Ryan McKinley --- pkg/services/searchV2/index.go | 3 +- pkg/services/searchV2/index_test.go | 82 +++++++++++++++++++++ pkg/services/searchV2/service_bench_test.go | 3 +- 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/pkg/services/searchV2/index.go b/pkg/services/searchV2/index.go index 27a3c5a8102..627b21903e0 100644 --- a/pkg/services/searchV2/index.go +++ b/pkg/services/searchV2/index.go @@ -849,7 +849,8 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or rows := make([]*dashboardQueryResult, 0) err := l.sql.WithDbSession(dashboardQueryCtx, func(sess *db.Session) error { sess.Table("dashboard"). - Where("org_id = ?", orgID) + Where("org_id = ?", orgID). + Where("deleted IS NULL") // don't index soft delete files if lastID > 0 { sess.Where("id > ?", lastID) diff --git a/pkg/services/searchV2/index_test.go b/pkg/services/searchV2/index_test.go index f0fa100dc91..b4cb38506f1 100644 --- a/pkg/services/searchV2/index_test.go +++ b/pkg/services/searchV2/index_test.go @@ -12,11 +12,21 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" + "github.com/grafana/grafana/pkg/services/dashboards/database" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/folder/foldertest" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/org/orgtest" + "github.com/grafana/grafana/pkg/services/quota/quotatest" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/services/store/entity" + "github.com/grafana/grafana/pkg/services/tag/tagimpl" + "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/setting" ) @@ -731,3 +741,75 @@ func TestDashboardIndex_MultiTermPrefixMatch(t *testing.T) { }) } } + +func setupIntegrationEnv(t *testing.T, folderCount, dashboardsPerFolder int, sqlStore *sqlstore.SQLStore) (*StandardSearchService, *user.SignedInUser, error) { + err := populateDB(folderCount, dashboardsPerFolder, sqlStore) + require.NoError(t, err, "error when populating the database for integration test") + + // load all dashboards and folders + dbLoadingBatchSize := (dashboardsPerFolder + 1) * folderCount + cfg := &setting.Cfg{Search: setting.SearchSettings{DashboardLoadingBatchSize: dbLoadingBatchSize}} + features := featuremgmt.WithFeatures() + orgSvc := &orgtest.FakeOrgService{ + ExpectedOrgs: []*org.OrgDTO{{ID: 1}}, + } + searchService, ok := ProvideService(cfg, sqlStore, store.NewDummyEntityEventsService(), actest.FakeService{}, + tracing.InitializeTracerForTest(), features, orgSvc, nil, foldertest.NewFakeService()).(*StandardSearchService) + require.True(t, ok) + + err = runSearchService(searchService) + require.NoError(t, err, "error when running search service for integration test") + + user := getSignedInUser(folderCount, dashboardsPerFolder) + + return searchService, user, nil +} + +func TestIntegrationSoftDeletion(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + // Set up search v2. + folderCount := 1 + dashboardsPerFolder := 1 + sqlStore, cfg := db.InitTestDBWithCfg(t) + searchService, testUser, err := setupIntegrationEnv(t, folderCount, dashboardsPerFolder, sqlStore) + require.NoError(t, err) + + // Query search v2 to ensure "dashboard2" is present. + result := searchService.doDashboardQuery(context.Background(), testUser, 1, DashboardQuery{Kind: []string{string(entityKindDashboard)}}) + require.NoError(t, result.Error) + require.NotZero(t, len(result.Frames)) + for _, field := range result.Frames[0].Fields { + if field.Name == "uid" { + require.Equal(t, dashboardsPerFolder, field.Len()) + break + } + } + + // Set up dashboard store. + quotaService := quotatest.New(false, nil) + featureToggles := featuremgmt.WithFeatures(featuremgmt.FlagPanelTitleSearch, featuremgmt.FlagDashboardRestore) + dashboardStore, err := database.ProvideDashboardStore(sqlStore, cfg, featureToggles, tagimpl.ProvideService(sqlStore), quotaService) + require.NoError(t, err) + + // Soft delete "dashboard2". + err = dashboardStore.SoftDeleteDashboard(context.Background(), 1, "dashboard2") + require.NoError(t, err) + + // Reindex to ensure "dashboard2" is excluded from the index. + searchService.dashboardIndex.reIndexFromScratch(context.Background()) + + // Query search v2 to ensure "dashboard2" is no longer present. + expectedResultCount := dashboardsPerFolder - 1 + result2 := searchService.doDashboardQuery(context.Background(), testUser, 1, DashboardQuery{Kind: []string{string(entityKindDashboard)}}) + require.NoError(t, result2.Error) + require.NotZero(t, len(result2.Frames)) + for _, field := range result2.Frames[0].Fields { + if field.Name == "uid" { + require.Equal(t, expectedResultCount, field.Len()) + break + } + } +} diff --git a/pkg/services/searchV2/service_bench_test.go b/pkg/services/searchV2/service_bench_test.go index 575fbf36f17..778e7bd2bf8 100644 --- a/pkg/services/searchV2/service_bench_test.go +++ b/pkg/services/searchV2/service_bench_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol/actest" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/folder/foldertest" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/org/orgtest" "github.com/grafana/grafana/pkg/services/store" @@ -41,7 +42,7 @@ func setupBenchEnv(b *testing.B, folderCount, dashboardsPerFolder int) (*Standar ExpectedOrgs: []*org.OrgDTO{{ID: 1}}, } searchService, ok := ProvideService(cfg, sqlStore, store.NewDummyEntityEventsService(), actest.FakeService{}, - tracing.InitializeTracerForTest(), features, orgSvc, nil, nil).(*StandardSearchService) + tracing.InitializeTracerForTest(), features, orgSvc, nil, foldertest.NewFakeService()).(*StandardSearchService) require.True(b, ok) err = runSearchService(searchService)