public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup of old search functionality (#110861)

Browse Source 
* Remove support for initMinSize.
Remove support for searchAfterWrite option, now it defaults to true.

* Remove reference to deprecated feature toggle.

* Remove feature toggle completely.

* Remove code related to indexing on watch events.

* Fix compilation error.

* Remove unused field.
This commit is contained in:
Peter Štibraný
2025-09-11 08:23:03 +00:00
committed by GitHub
parent ecf08ad7d5
commit 6fa6a5708a
23 changed files with 164 additions and 1054 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/search/bleve.go
+5 -31
View File
@@ -213,8 +213,7 @@ func newBleveIndex(path string, mapper mapping.IndexMapping) (bleve.Index, error
// BuildIndex builds an index from scratch or retrieves it from the filesystem.
// If built successfully, the new index replaces the old index in the cache (if there was any).
// An index in the file system is considered to be valid if the requested resourceVersion is smaller than or equal to
// the resourceVersion used to build the index and the number of indexed objects matches the expected size.
// Existing index in the file system is reused, if it exists, and if size indicates that we should use file-based index, and rebuild is not true.
// The return value of "builder" should be the RV returned from List. This will be stored as the index RV
//
//nolint:gocyclo
@@ -222,13 +221,11 @@ func (b *bleveBackend) BuildIndex(
ctx context.Context,
key resource.NamespacedResource,
size int64,
resourceVersion int64,
fields resource.SearchableDocumentFields,
indexBuildReason string,
builder resource.BuildFn,
updater resource.UpdateFn,
rebuild bool,
searchAfterWrite bool,
) (resource.ResourceIndex, error) {
_, span := b.tracer.Start(ctx, tracingPrexfixBleve+"BuildIndex")
defer span.End()
@@ -238,7 +235,6 @@ func (b *bleveBackend) BuildIndex(
attribute.String("group", key.Group),
attribute.String("resource", key.Resource),
attribute.Int64("size", size),
attribute.Int64("rv", resourceVersion),
attribute.String("reason", indexBuildReason),
)
@@ -254,7 +250,7 @@ func (b *bleveBackend) BuildIndex(
return nil, err
}
logWithDetails := b.log.With("namespace", key.Namespace, "group", key.Group, "resource", key.Resource, "size", size, "rv", resourceVersion, "reason", indexBuildReason)
logWithDetails := b.log.With("namespace", key.Namespace, "group", key.Group, "resource", key.Resource, "size", size, "reason", indexBuildReason)
// Close the newly created/opened index by default.
closeIndex := true
@@ -290,8 +286,8 @@ func (b *bleveBackend) BuildIndex(
// We only check for the existing file-based index if we don't already have an open index for this key.
// This happens on startup, or when memory-based index has expired. (We don't expire file-based indexes)
// If we do have an unexpired cached index already, we always build a new index from scratch.
if cachedIndex == nil && resourceVersion > 0 && !rebuild {
index, fileIndexName, indexRV = b.findPreviousFileBasedIndex(resourceDir, resourceVersion, size, searchAfterWrite)
if cachedIndex == nil && !rebuild {
index, fileIndexName, indexRV = b.findPreviousFileBasedIndex(resourceDir)
}
if index != nil {
@@ -509,7 +505,7 @@ func formatIndexName(now time.Time) string {
return now.Format("20060102-150405")
}
func (b *bleveBackend) findPreviousFileBasedIndex(resourceDir string, resourceVersion int64, size int64, searchAfterWrite bool) (bleve.Index, string, int64) {
func (b *bleveBackend) findPreviousFileBasedIndex(resourceDir string) (bleve.Index, string, int64) {
entries, err := os.ReadDir(resourceDir)
if err != nil {
return nil, "", 0
@@ -528,21 +524,6 @@ func (b *bleveBackend) findPreviousFileBasedIndex(resourceDir string, resourceVe
continue
}
if !searchAfterWrite {
cnt, err := idx.DocCount()
if err != nil {
b.log.Debug("error getting count from index", "indexDir", indexDir, "err", err)
_ = idx.Close()
continue
}
if uint64(size) != cnt {
b.log.Debug("index count mismatch. ignoring index", "indexDir", indexDir, "size", size, "cnt", cnt)
_ = idx.Close()
continue
}
}
indexRV, err := getRV(idx)
if err != nil {
b.log.Error("error getting rv from index", "indexDir", indexDir, "err", err)
@@ -552,13 +533,6 @@ func (b *bleveBackend) findPreviousFileBasedIndex(resourceDir string, resourceVe
continue
}
// if searchAfterWrite is enabled, we don't need to re-build the index, as it will be updated at request time
if !searchAfterWrite && indexRV < resourceVersion {
b.log.Debug("indexRV is less than requested resourceVersion. ignoring index", "indexDir", indexDir, "rv", indexRV, "resourceVersion", resourceVersion)
_ = idx.Close()
continue
}
return idx, indexName, indexRV
}
pkg/storage/unified/search/bleve_search_test.go
+1 -2
View File
@@ -237,7 +237,6 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize
t.Cleanup(backend.CloseAllIndexes)
rv := int64(10)
ctx := identity.WithRequester(context.Background(), &user.SignedInUser{Namespace: "ns"})
info, err := search.DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) {
@@ -254,7 +253,7 @@ func newTestDashboardsIndex(t testing.TB, threshold int64, size int64, batchSize
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
}, size, rv, info.Fields, "test", writer, nil, false, false)
}, size, info.Fields, "test", writer, nil, false)
require.NoError(t, err)
return index
pkg/storage/unified/search/bleve_test.go
+82 -97
View File
@@ -89,7 +89,7 @@ func TestBleveBackend(t *testing.T) {
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
}, 2, rv, info.Fields, "test", func(index resource.ResourceIndex) (int64, error) {
}, 2, info.Fields, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -192,7 +192,7 @@ func TestBleveBackend(t *testing.T) {
return 0, err
}
return rv, nil
}, nil, false, false)
}, nil, false)
require.NoError(t, err)
require.NotNil(t, index)
dashboardsIndex = index
@@ -370,7 +370,7 @@ func TestBleveBackend(t *testing.T) {
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
}, 2, rv, fields, "test", func(index resource.ResourceIndex) (int64, error) {
}, 2, fields, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -421,7 +421,7 @@ func TestBleveBackend(t *testing.T) {
return 0, err
}
return rv, nil
}, nil, false, false)
}, nil, false)
require.NoError(t, err)
require.NotNil(t, index)
foldersIndex = index
@@ -786,7 +786,7 @@ func TestBuildIndexExpiration(t *testing.T) {
t.Run("memory based indexes should expire", func(t *testing.T) {
backend, reg := setupBleveBackend(t, 5, time.Nanosecond, "")
builtIndex, err := backend.BuildIndex(context.Background(), ns, 1 /* below FileThreshold */, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false, false)
builtIndex, err := backend.BuildIndex(context.Background(), ns, 1 /* below FileThreshold */, nil, "test", indexTestDocs(ns, 1, 100), nil, false)
require.NoError(t, err)
// Wait for index expiration, which is 1ns
@@ -807,7 +807,7 @@ func TestBuildIndexExpiration(t *testing.T) {
backend, reg := setupBleveBackend(t, 5, time.Nanosecond, "")
// size=100 is above FileThreshold, this will be file-based index
builtIndex, err := backend.BuildIndex(context.Background(), ns, 100, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false, false)
builtIndex, err := backend.BuildIndex(context.Background(), ns, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false)
require.NoError(t, err)
// Wait for index expiration, which is 1ns
@@ -839,9 +839,9 @@ func TestCloseAllIndexes(t *testing.T) {
tmpDir := t.TempDir()
backend1, reg := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
_, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, false)
_, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, nil, "test", indexTestDocs(ns, 10, 100), nil, false)
require.NoError(t, err)
_, err = backend1.BuildIndex(context.Background(), ns2, 1 /* memory based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, false)
_, err = backend1.BuildIndex(context.Background(), ns2, 1 /* memory based */, nil, "test", indexTestDocs(ns, 10, 100), nil, false)
require.NoError(t, err)
// Verify two open indexes.
@@ -865,91 +865,76 @@ func TestBuildIndex(t *testing.T) {
RVBiggerThan RV = "more"
RVSame RV = "same"
)
for _, searchAfterWrite := range []bool{false, true} {
for _, rebuild := range []bool{false, true} {
for _, sameSize := range []bool{false, true} {
for _, documentRV := range []RV{RVLessThan, RVSame, RVBiggerThan} {
shouldRebuild := false
if rebuild {
shouldRebuild = true
} else if !searchAfterWrite && !sameSize {
shouldRebuild = true
} else if !searchAfterWrite && documentRV == RVBiggerThan {
shouldRebuild = true
}
for _, rebuild := range []bool{false, true} {
for _, sameSize := range []bool{false, true} {
for _, documentRV := range []RV{RVLessThan, RVSame, RVBiggerThan} {
shouldRebuild := rebuild
testName := ""
if shouldRebuild {
testName += "should NOT reuse index "
} else {
testName += "should reuse index "
}
if sameSize {
testName += "on same size "
} else {
testName += "on different size "
}
switch documentRV {
case RVLessThan:
testName += "and documentRV < indexRV "
case RVBiggerThan:
testName += "and documentRV > indexRV "
case RVSame:
testName += "and documentRV = indexRV "
}
if rebuild {
testName += "when rebuild is true "
} else {
testName += "when rebuild is false "
}
if searchAfterWrite {
testName += "and searchAfterWrite is true"
} else {
testName += "and searchAfterWrite is false"
}
t.Run(testName, func(t *testing.T) {
tmpDir := t.TempDir()
var size int64 = 10
var rv int64 = 100
backend1, _ := createBleveBackendAndIndex(t, tmpDir, ns, size, rv, 10, rebuild, searchAfterWrite)
backend1.CloseAllIndexes()
if !sameSize {
size = 11
}
switch documentRV {
case RVBiggerThan:
rv = 101
case RVLessThan:
rv = 99
case RVSame:
}
backend2, idx := createBleveBackendAndIndex(t, tmpDir, ns, size, rv, 1000, rebuild, searchAfterWrite)
cnt, err := idx.DocCount(context.Background(), "")
require.NoError(t, err)
if shouldRebuild {
require.Equal(t, int64(1000), cnt, "Index has been not rebuilt")
} else {
require.Equal(t, int64(10), cnt, "Index has not been reused")
}
backend2.CloseAllIndexes()
})
testName := ""
if shouldRebuild {
testName += "should NOT reuse index "
} else {
testName += "should reuse index "
}
if sameSize {
testName += "on same size "
} else {
testName += "on different size "
}
switch documentRV {
case RVLessThan:
testName += "and documentRV < indexRV "
case RVBiggerThan:
testName += "and documentRV > indexRV "
case RVSame:
testName += "and documentRV = indexRV "
}
if rebuild {
testName += "when rebuild is true "
} else {
testName += "when rebuild is false "
}
t.Run(testName, func(t *testing.T) {
tmpDir := t.TempDir()
var size int64 = 10
var rv int64 = 100
backend1, _ := createBleveBackendAndIndex(t, tmpDir, ns, size, rv, 10, rebuild)
backend1.CloseAllIndexes()
if !sameSize {
size = 11
}
switch documentRV {
case RVBiggerThan:
rv = 101
case RVLessThan:
rv = 99
case RVSame:
}
backend2, idx := createBleveBackendAndIndex(t, tmpDir, ns, size, rv, 1000, rebuild)
cnt, err := idx.DocCount(context.Background(), "")
require.NoError(t, err)
if shouldRebuild {
require.Equal(t, int64(1000), cnt, "Index has been not rebuilt")
} else {
require.Equal(t, int64(10), cnt, "Index has not been reused")
}
backend2.CloseAllIndexes()
})
}
}
}
}
func createBleveBackendAndIndex(t *testing.T, tmpDir string, ns resource.NamespacedResource, size, rv int64, docCount int, rebuild, searchAfterWrite bool) (*bleveBackend, resource.ResourceIndex) {
func createBleveBackendAndIndex(t *testing.T, tmpDir string, ns resource.NamespacedResource, size, rv int64, docCount int, rebuild bool) (*bleveBackend, resource.ResourceIndex) {
backend, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
idx, err := backend.BuildIndex(context.Background(), ns, size /* file based */, rv, nil, "test", indexTestDocs(ns, docCount, rv), nil, rebuild, searchAfterWrite)
idx, err := backend.BuildIndex(context.Background(), ns, size /* file based */, nil, "test", indexTestDocs(ns, docCount, rv), nil, rebuild)
require.NoError(t, err)
return backend, idx
}
@@ -977,7 +962,7 @@ func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) {
if testCase.firstInMemory {
firstSize = 1
}
firstIndex, err := backend.BuildIndex(context.Background(), ns, int64(firstSize), 100, nil, "test", indexTestDocs(ns, firstSize, 100), nil, false, false)
firstIndex, err := backend.BuildIndex(context.Background(), ns, int64(firstSize), nil, "test", indexTestDocs(ns, firstSize, 100), nil, false)
require.NoError(t, err)
if testCase.firstInMemory {
@@ -993,7 +978,7 @@ func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) {
secondSize = 1
openInMemoryIndexes = 1
}
secondIndex, err := backend.BuildIndex(context.Background(), ns, int64(secondSize), 100, nil, "test", indexTestDocs(ns, secondSize, 100), nil, false, false)
secondIndex, err := backend.BuildIndex(context.Background(), ns, int64(secondSize), nil, "test", indexTestDocs(ns, secondSize, 100), nil, false)
require.NoError(t, err)
if testCase.secondInMemory {
@@ -1142,13 +1127,13 @@ func testBleveIndexWithFailures(t *testing.T, fileBased bool) {
// size=100 is above FileThreshold (5), make it a file-based index.
size = 100
}
_, err := backend.BuildIndex(context.Background(), ns, size, 100, nil, "test", func(index resource.ResourceIndex) (int64, error) {
_, err := backend.BuildIndex(context.Background(), ns, size, nil, "test", func(index resource.ResourceIndex) (int64, error) {
return 0, fmt.Errorf("fail")
}, nil, false, false)
}, nil, false)
require.Error(t, err)
// Even though previous build of the index failed, new building of the index should work.
_, err = backend.BuildIndex(context.Background(), ns, size, 100, nil, "test", indexTestDocs(ns, int(size), 100), nil, false, false)
_, err = backend.BuildIndex(context.Background(), ns, size, nil, "test", indexTestDocs(ns, int(size), 100), nil, false)
require.NoError(t, err)
}
@@ -1160,7 +1145,7 @@ func TestIndexUpdate(t *testing.T) {
}
be, _ := setupBleveBackend(t, 5, 1*time.Minute, "")
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false, false)
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false)
require.NoError(t, err)
resp := searchTitle(t, idx, "gen", 10, ns)
@@ -1214,7 +1199,7 @@ func TestConcurrentIndexUpdateAndBuildIndex(t *testing.T) {
return sinceRV + int64(5), 5, err
}
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), updaterFn, false, false)
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, nil, "test", indexTestDocs(ns, 10, 100), updaterFn, false)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
@@ -1222,7 +1207,7 @@ func TestConcurrentIndexUpdateAndBuildIndex(t *testing.T) {
_, err = idx.UpdateIndex(ctx, "test")
require.NoError(t, err)
_, err = be.BuildIndex(t.Context(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), updaterFn, false, false)
_, err = be.BuildIndex(t.Context(), ns, 10 /* file based */, nil, "test", indexTestDocs(ns, 10, 100), updaterFn, false)
require.NoError(t, err)
_, err = idx.UpdateIndex(ctx, "test")
@@ -1238,7 +1223,7 @@ func TestConcurrentIndexUpdateSearchAndRebuild(t *testing.T) {
be, _ := setupBleveBackend(t, 5, 1*time.Minute, "")
_, err := be.BuildIndex(t.Context(), ns, 10, 0, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false, false)
_, err := be.BuildIndex(t.Context(), ns, 10, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false)
require.NoError(t, err)
wg := sync.WaitGroup{}
@@ -1301,7 +1286,7 @@ func TestConcurrentIndexUpdateSearchAndRebuild(t *testing.T) {
go func() {
defer wg.Done()
for ctx.Err() == nil {
_, err := be.BuildIndex(t.Context(), ns, 10, 0, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false, false)
_, err := be.BuildIndex(t.Context(), ns, 10, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false)
require.NoError(t, err)
rebuilds.Inc()
}
@@ -1324,7 +1309,7 @@ func TestConcurrentIndexUpdateAndSearch(t *testing.T) {
be, _ := setupBleveBackend(t, 5, 1*time.Minute, "")
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false, false)
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, nil, "test", indexTestDocs(ns, 10, 100), updateTestDocs(ns, 5), false)
require.NoError(t, err)
wg := sync.WaitGroup{}
@@ -1388,7 +1373,7 @@ func TestIndexUpdateWithErrors(t *testing.T) {
time.Sleep(100 * time.Millisecond)
return 0, 0, updateErr
}
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), updaterFn, false, false)
idx, err := be.BuildIndex(t.Context(), ns, 10 /* file based */, nil, "test", indexTestDocs(ns, 10, 100), updaterFn, false)
require.NoError(t, err)
t.Run("update fail", func(t *testing.T) {
pkg/storage/unified/search/options.go
-1
View File
@@ -39,7 +39,6 @@ func NewSearchOptions(features featuremgmt.FeatureToggles, cfg *setting.Cfg, tra
Resources: docs,
WorkerThreads: cfg.IndexWorkers,
InitMinCount: cfg.IndexMinCount,
InitMaxCount: cfg.IndexMaxCount,
RebuildInterval: cfg.IndexRebuildInterval,
}, nil
}