feat(unified): add index scoring model config (#116210)

* feat(unified): add bm25 index scoring model

We want try BM25 scoring model since they have global scoring which we can probably re-use for fan-in/fan-out logic

https://github.com/blevesearch/bleve/blob/32d98823c4b7482c62cc6c847508ed7659c23c37/docs/scoring.md#global-scoring

* fix(plugins): update plugin test data

pkg/storage/unified/search/bleve.go

@@ -81,6 +81,11 @@ type BleveOptions struct {
 	// Indexes that are not owned by current instance are eligible for cleanup.
 	// If nil, all indexes are owned by the current instance.
 	OwnsIndex func(key resource.NamespacedResource) (bool, error)
+
+	// ScoringModel defines the scoring model used for the bleve indexes
+	// Default: index.TFIDFScoring
+	// Supported values: index.TFIDFScoring and index.BM25Scoring
+	ScoringModel string
 }
 
 type bleveBackend struct {
@@ -368,7 +373,7 @@ func (b *bleveBackend) BuildIndex(
 		attribute.String("reason", indexBuildReason),
 	)
 
-	mapper, err := GetBleveMappings(fields)
+	mapper, err := GetBleveMappings(b.opts.ScoringModel, fields)
 	if err != nil {
 		return nil, err
 	}

pkg/storage/unified/search/bleve_integration_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	index "github.com/blevesearch/bleve_index_api"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
@@ -19,6 +20,7 @@ func TestBleveSearchBackend(t *testing.T) {
 		backend, err := NewBleveBackend(BleveOptions{
 			Root:          tempDir,
 			FileThreshold: 5,
+			ScoringModel:  index.BM25Scoring,
 		}, nil)
 		require.NoError(t, err)
 		require.NotNil(t, backend)
@@ -52,3 +54,32 @@ func TestSearchBackendBenchmark(t *testing.T) {
 
 	unitest.BenchmarkSearchBackend(t, backend, opts)
 }
+
+func BenchmarkScoringModels(b *testing.B) {
+	models := []string{index.TFIDFScoring, index.BM25Scoring}
+
+	for _, model := range models {
+		b.Run(model, func(b *testing.B) {
+			tempDir := b.TempDir()
+
+			backend, err := NewBleveBackend(BleveOptions{
+				Root:         tempDir,
+				ScoringModel: model,
+			}, nil)
+			require.NoError(b, err)
+			require.NotNil(b, backend)
+
+			b.Cleanup(backend.Stop)
+
+			opts := &unitest.BenchmarkOptions{
+				NumResources:     1000,
+				Concurrency:      4,
+				NumNamespaces:    10,
+				NumGroups:        10,
+				NumResourceTypes: 10,
+			}
+
+			unitest.BenchmarkSearchBackend(b, backend, opts)
+		})
+	}
+}

pkg/storage/unified/search/bleve_mappings.go

@@ -5,13 +5,15 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
 	"github.com/blevesearch/bleve/v2/mapping"
-
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
 	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
 )
 
-func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
+func GetBleveMappings(scoringModel string, fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
 	mapper := bleve.NewIndexMapping()
+	if scoringModel != "" {
+		mapper.ScoringModel = scoringModel
+	}
 
 	err := RegisterCustomAnalyzers(mapper)
 	if err != nil {

pkg/storage/unified/search/bleve_mappings_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestDocumentMapping(t *testing.T) {
-	mappings, err := search.GetBleveMappings(nil)
+	mappings, err := search.GetBleveMappings("", nil)
 	require.NoError(t, err)
 	data := resource.IndexableDocument{
 		Title:       "title",

pkg/storage/unified/search/bleve_search_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/blevesearch/bleve/v2"
+	index "github.com/blevesearch/bleve_index_api"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
@@ -258,6 +259,7 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer re
 	backend, err := search.NewBleveBackend(search.BleveOptions{
 		Root:          t.TempDir(),
 		FileThreshold: threshold, // use in-memory for tests
+		ScoringModel:  index.BM25Scoring,
 	}, nil)
 	require.NoError(t, err)
 

pkg/storage/unified/search/bleve_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/blevesearch/bleve/v2"
+	index "github.com/blevesearch/bleve_index_api"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
@@ -50,6 +51,7 @@ func TestBleveBackend(t *testing.T) {
 	backend, err := NewBleveBackend(BleveOptions{
 		Root:          tmpdir,
 		FileThreshold: 5, // with more than 5 items we create a file on disk
+		ScoringModel:  index.BM25Scoring,
 	}, nil)
 	require.NoError(t, err)
 	t.Cleanup(backend.Stop)
@@ -773,6 +775,7 @@ func setupBleveBackend(t *testing.T, options ...setupOption) (*bleveBackend, pro
 		IndexCacheTTL: defaultIndexCacheTTL,
 		Logger:        log.NewNopLogger(),
 		BuildVersion:  buildVersion,
+		ScoringModel:  index.BM25Scoring,
 	}
 	for _, opt := range options {
 		opt(&opts)

pkg/storage/unified/search/options.go

@@ -46,6 +46,7 @@ func NewSearchOptions(
 			BuildVersion:           cfg.BuildVersion,
 			OwnsIndex:              ownsIndexFn,
 			IndexMinUpdateInterval: cfg.IndexMinUpdateInterval,
+			ScoringModel:           cfg.IndexScoringModel,
 		}, indexMetrics)
 
 		if err != nil {

pkg/storage/unified/sql/test/integration_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	index "github.com/blevesearch/bleve_index_api"
 	"github.com/go-jose/go-jose/v4/jwt"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
@@ -129,21 +130,28 @@ func TestIntegrationSearchAndStorage(t *testing.T) {
 
 	ctx := context.Background()
 
-	// Create a new bleve backend
-	search, err := search.NewBleveBackend(search.BleveOptions{
-		FileThreshold: 0,
-		Root:          t.TempDir(),
-	}, nil)
-	require.NoError(t, err)
-	require.NotNil(t, search)
-	t.Cleanup(search.Stop)
+	scoringModels := []string{index.TFIDFScoring, index.BM25Scoring}
 
-	// Create a new resource backend
-	storage, _ := newTestBackend(t, false, 0)
-	require.NotNil(t, storage)
+	for _, model := range scoringModels {
+		t.Run(model, func(t *testing.T) {
+			// Create a new bleve backend
+			search, err := search.NewBleveBackend(search.BleveOptions{
+				FileThreshold: 0,
+				Root:          t.TempDir(),
+				ScoringModel:  model,
+			}, nil)
+			require.NoError(t, err)
+			require.NotNil(t, search)
+			t.Cleanup(search.Stop)
 
-	// Run the shared storage and search tests
-	unitest.RunTestSearchAndStorage(t, ctx, storage, search)
+			// Create a new resource backend
+			storage, _ := newTestBackend(t, false, 0)
+			require.NotNil(t, storage)
+
+			// Run the shared storage and search tests
+			unitest.RunTestSearchAndStorage(t, ctx, storage, search)
+		})
+	}
 }
 
 func TestClientServer(t *testing.T) {