public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(unified): add index scoring model config (#116210)

Browse Source 
* feat(unified): add bm25 index scoring model

We want try BM25 scoring model since they have global scoring which we can probably re-use for fan-in/fan-out logic

https://github.com/blevesearch/bleve/blob/32d98823c4b7482c62cc6c847508ed7659c23c37/docs/scoring.md#global-scoring

* fix(plugins): update plugin test data
This commit is contained in:
Rafael Bortolon Paulovic
2026-01-14 12:07:53 +01:00
committed by GitHub
parent 170ac31c5a
commit 987c1fc6b6
11 changed files with 89 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/setting/setting.go
+1
View File
@@ -600,6 +600,7 @@ type Cfg struct {
IndexRebuildInterval time.Duration
IndexCacheTTL time.Duration
IndexMinUpdateInterval time.Duration // Don't update index if it was updated less than this interval ago.
IndexScoringModel string // Note: Temporary config to switch the index scoring model and will be removed soon.
MaxFileIndexAge time.Duration // Max age of file-based indexes. Index older than this will be rebuilt asynchronously.
MinFileIndexBuildVersion string // Minimum version of Grafana that built the file-based index. If index was built with older Grafana, it will be rebuilt asynchronously.
EnableSharding bool
pkg/setting/setting_unified_storage.go
+4
View File
@@ -123,6 +123,10 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
cfg.IndexRebuildInterval = section.Key("index_rebuild_interval").MustDuration(24 * time.Hour)
cfg.IndexCacheTTL = section.Key("index_cache_ttl").MustDuration(10 * time.Minute)
cfg.IndexMinUpdateInterval = section.Key("index_min_update_interval").MustDuration(0)
cfg.IndexScoringModel = section.Key("index_scoring_model").MustString("")
if cfg.IndexScoringModel != "" {
cfg.Logger.Info("Index scoring model set", "model", cfg.IndexScoringModel)
}
cfg.SprinklesApiServer = section.Key("sprinkles_api_server").String()
cfg.SprinklesApiServerPageLimit = section.Key("sprinkles_api_server_page_limit").MustInt(10000)
cfg.CACertPath = section.Key("ca_cert_path").String()
pkg/storage/unified/search/bleve.go
+6 -1
View File
@@ -81,6 +81,11 @@ type BleveOptions struct {
// Indexes that are not owned by current instance are eligible for cleanup.
// If nil, all indexes are owned by the current instance.
OwnsIndex func(key resource.NamespacedResource) (bool, error)
// ScoringModel defines the scoring model used for the bleve indexes
// Default: index.TFIDFScoring
// Supported values: index.TFIDFScoring and index.BM25Scoring
ScoringModel string
}
type bleveBackend struct {
@@ -368,7 +373,7 @@ func (b *bleveBackend) BuildIndex(
attribute.String("reason", indexBuildReason),
)
mapper, err := GetBleveMappings(fields)
mapper, err := GetBleveMappings(b.opts.ScoringModel, fields)
if err != nil {
return nil, err
}
pkg/storage/unified/search/bleve_integration_test.go
+31
View File
@@ -4,6 +4,7 @@ import (
"context"
"testing"
index "github.com/blevesearch/bleve_index_api"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
@@ -19,6 +20,7 @@ func TestBleveSearchBackend(t *testing.T) {
backend, err := NewBleveBackend(BleveOptions{
Root: tempDir,
FileThreshold: 5,
ScoringModel: index.BM25Scoring,
}, nil)
require.NoError(t, err)
require.NotNil(t, backend)
@@ -52,3 +54,32 @@ func TestSearchBackendBenchmark(t *testing.T) {
unitest.BenchmarkSearchBackend(t, backend, opts)
}
func BenchmarkScoringModels(b *testing.B) {
models := []string{index.TFIDFScoring, index.BM25Scoring}
for _, model := range models {
b.Run(model, func(b *testing.B) {
tempDir := b.TempDir()
backend, err := NewBleveBackend(BleveOptions{
Root: tempDir,
ScoringModel: model,
}, nil)
require.NoError(b, err)
require.NotNil(b, backend)
b.Cleanup(backend.Stop)
opts := &unitest.BenchmarkOptions{
NumResources: 1000,
Concurrency: 4,
NumNamespaces: 10,
NumGroups: 10,
NumResourceTypes: 10,
}
unitest.BenchmarkSearchBackend(b, backend, opts)
})
}
}
pkg/storage/unified/search/bleve_mappings.go
+4 -2
View File
@@ -5,13 +5,15 @@ import (
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
func GetBleveMappings(scoringModel string, fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
mapper := bleve.NewIndexMapping()
if scoringModel != "" {
mapper.ScoringModel = scoringModel
}
err := RegisterCustomAnalyzers(mapper)
if err != nil {
pkg/storage/unified/search/bleve_mappings_test.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
)
func TestDocumentMapping(t *testing.T) {
mappings, err := search.GetBleveMappings(nil)
mappings, err := search.GetBleveMappings("", nil)
require.NoError(t, err)
data := resource.IndexableDocument{
Title: "title",
pkg/storage/unified/search/bleve_search_test.go
+2
View File
@@ -7,6 +7,7 @@ import (
"testing"
"github.com/blevesearch/bleve/v2"
index "github.com/blevesearch/bleve_index_api"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/apimachinery/identity"
@@ -258,6 +259,7 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, writer re
backend, err := search.NewBleveBackend(search.BleveOptions{
Root: t.TempDir(),
FileThreshold: threshold, // use in-memory for tests
ScoringModel: index.BM25Scoring,
}, nil)
require.NoError(t, err)
pkg/storage/unified/search/bleve_test.go
+3
View File
@@ -14,6 +14,7 @@ import (
"time"
"github.com/blevesearch/bleve/v2"
index "github.com/blevesearch/bleve_index_api"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/assert"
@@ -50,6 +51,7 @@ func TestBleveBackend(t *testing.T) {
backend, err := NewBleveBackend(BleveOptions{
Root: tmpdir,
FileThreshold: 5, // with more than 5 items we create a file on disk
ScoringModel: index.BM25Scoring,
}, nil)
require.NoError(t, err)
t.Cleanup(backend.Stop)
@@ -773,6 +775,7 @@ func setupBleveBackend(t *testing.T, options ...setupOption) (*bleveBackend, pro
IndexCacheTTL: defaultIndexCacheTTL,
Logger: log.NewNopLogger(),
BuildVersion: buildVersion,
ScoringModel: index.BM25Scoring,
}
for _, opt := range options {
opt(&opts)
pkg/storage/unified/search/options.go
+1
View File
@@ -46,6 +46,7 @@ func NewSearchOptions(
BuildVersion: cfg.BuildVersion,
OwnsIndex: ownsIndexFn,
IndexMinUpdateInterval: cfg.IndexMinUpdateInterval,
ScoringModel: cfg.IndexScoringModel,
}, indexMetrics)
if err != nil {
pkg/storage/unified/sql/test/integration_test.go
+21 -13
View File
@@ -6,6 +6,7 @@ import (
"testing"
"time"
index "github.com/blevesearch/bleve_index_api"
"github.com/go-jose/go-jose/v4/jwt"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
@@ -129,21 +130,28 @@ func TestIntegrationSearchAndStorage(t *testing.T) {
ctx := context.Background()
// Create a new bleve backend
search, err := search.NewBleveBackend(search.BleveOptions{
FileThreshold: 0,
Root: t.TempDir(),
}, nil)
require.NoError(t, err)
require.NotNil(t, search)
t.Cleanup(search.Stop)
scoringModels := []string{index.TFIDFScoring, index.BM25Scoring}
// Create a new resource backend
storage, _ := newTestBackend(t, false, 0)
require.NotNil(t, storage)
for _, model := range scoringModels {
t.Run(model, func(t *testing.T) {
// Create a new bleve backend
search, err := search.NewBleveBackend(search.BleveOptions{
FileThreshold: 0,
Root: t.TempDir(),
ScoringModel: model,
}, nil)
require.NoError(t, err)
require.NotNil(t, search)
t.Cleanup(search.Stop)
// Run the shared storage and search tests
unitest.RunTestSearchAndStorage(t, ctx, storage, search)
// Create a new resource backend
storage, _ := newTestBackend(t, false, 0)
require.NotNil(t, storage)
// Run the shared storage and search tests
unitest.RunTestSearchAndStorage(t, ctx, storage, search)
})
}
}
func TestClientServer(t *testing.T) {
pkg/tests/api/plugins/data/expectedListResp.json
+15 -15
View File
@@ -209,7 +209,7 @@
"path": "public/plugins/grafana-azure-monitor-datasource/img/azure_monitor_cpu.png"
}
],
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": [
"azure",
@@ -589,7 +589,7 @@
"hasUpdate": false,
"defaultNavUrl": "/plugins/datagrid/",
"category": "",
"state": "beta",
"state": "deprecated",
"signature": "internal",
"signatureType": "",
"signatureOrg": "",
@@ -880,7 +880,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -934,7 +934,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": [
"grafana",
@@ -1000,7 +1000,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -1217,7 +1217,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -1325,7 +1325,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -1375,7 +1375,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -1425,7 +1425,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -1575,7 +1575,7 @@
},
"build": {},
"screenshots": null,
"version": "",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -1629,7 +1629,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": [
"grafana",
@@ -1734,7 +1734,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -2042,7 +2042,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -2092,7 +2092,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},
@@ -2445,7 +2445,7 @@
},
"build": {},
"screenshots": null,
"version": "12.3.0-pre",
"version": "12.4.0-pre",
"updated": "",
"keywords": null
},