search: add index batching (#104163)

* add basic search backend integration tests

* add search backend benchmark

* add benchmark indexServer

* fix

* lint

* add more tests

* lint

* do not use the poller

* batch write

* refactor and add tests

* improvements

* improvements

* cleanup

* only observe index success

* add monitorIndexEvents method

* nit use switch instead of if

* make newIndexQueueProcessor private

* simplify runProcessor

* go lint

pkg/storage/unified/sql/test/benchmark_test.go

@@ -2,20 +2,24 @@ package test
 
 import (
 	"context"
+	"os"
 	"testing"
 	"time"
 
-	infraDB "github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
+	"github.com/grafana/grafana/pkg/storage/unified/search"
 	"github.com/grafana/grafana/pkg/storage/unified/sql"
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
 	test "github.com/grafana/grafana/pkg/storage/unified/testing"
 	"github.com/stretchr/testify/require"
 )
 
-func newTestBackend(b *testing.B) resource.StorageBackend {
-	dbstore := infraDB.InitTestDB(b)
+func newTestBackend(b testing.TB) resource.StorageBackend {
+	dbstore := db.InitTestDB(b)
 	eDB, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), nil)
 	require.NoError(b, err)
 	require.NotNil(b, eDB)
@@ -32,10 +36,62 @@ func newTestBackend(b *testing.B) resource.StorageBackend {
 	return backend
 }
 
-func BenchmarkSQLStorageBackend(b *testing.B) {
+func TestIntegrationBenchmarkSQLStorageBackend(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+	if db.IsTestDBSpanner() {
+		t.Skip("Skipping benchmark on Spanner")
+	}
 	opts := test.DefaultBenchmarkOptions()
-	if infraDB.IsTestDbSQLite() {
+	if db.IsTestDbSQLite() {
 		opts.Concurrency = 1 // to avoid SQLite database is locked error
 	}
-	test.BenchmarkStorageBackend(b, newTestBackend(b), opts)
+	test.BenchmarkStorageBackend(t, newTestBackend(t), opts)
+}
+
+func TestIntegrationBenchmarkResourceServer(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+	if db.IsTestDBSpanner() {
+		t.Skip("Skipping benchmark on Spanner")
+	}
+
+	ctx := context.Background()
+	opts := &test.BenchmarkOptions{
+		NumResources:     1000,
+		Concurrency:      10,
+		NumNamespaces:    1,
+		NumGroups:        1,
+		NumResourceTypes: 1,
+	}
+	tempDir := t.TempDir()
+	t.Cleanup(func() {
+		_ = os.RemoveAll(tempDir)
+	})
+	// Create a new bleve backend
+	search, err := search.NewBleveBackend(search.BleveOptions{
+		Root: tempDir,
+	}, tracing.NewNoopTracerService(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorageSearchPermissionFiltering), nil)
+	require.NoError(t, err)
+	require.NotNil(t, search)
+
+	// Create a new resource backend
+	dbstore := db.InitTestDB(t)
+	eDB, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), nil)
+	require.NoError(t, err)
+	require.NotNil(t, eDB)
+
+	storage, err := sql.NewBackend(sql.BackendOptions{
+		DBProvider: eDB,
+		IsHA:       false,
+	})
+	require.NoError(t, err)
+	require.NotNil(t, storage)
+
+	err = storage.Init(ctx)
+	require.NoError(t, err)
+
+	test.BenchmarkIndexServer(t, ctx, storage, search, opts)
 }