From 18bc69f5c69811e125564107dc041b14e228de09 Mon Sep 17 00:00:00 2001 From: Will Assis <35489495+gassiss@users.noreply.github.com> Date: Fri, 29 Aug 2025 11:35:31 -0400 Subject: [PATCH] unified-storage: Bleve test cleanup (#110240) * consolidate index build tests * shut up logging in test --- pkg/storage/unified/search/bleve.go | 9 +- pkg/storage/unified/search/bleve_test.go | 599 ++++++----------------- 2 files changed, 145 insertions(+), 463 deletions(-) diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 6a22a96590f..ed9c64735dd 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -64,6 +64,8 @@ type BleveOptions struct { // Index cache TTL for bleve indices. 0 disables expiration for in-memory indexes. IndexCacheTTL time.Duration + + Logger *slog.Logger } type bleveBackend struct { @@ -96,8 +98,13 @@ func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, features featuremgm return nil, fmt.Errorf("bleve root is configured against a file (not folder)") } + log := opts.Logger + if log == nil { + log = slog.Default().With("logger", "bleve-backend") + } + be := &bleveBackend{ - log: slog.Default().With("logger", "bleve-backend"), + log: log, tracer: tracer, cache: map[resource.NamespacedResource]*bleveIndex{}, opts: opts, diff --git a/pkg/storage/unified/search/bleve_test.go b/pkg/storage/unified/search/bleve_test.go index 9a49c245a88..234e67af1c5 100644 --- a/pkg/storage/unified/search/bleve_test.go +++ b/pkg/storage/unified/search/bleve_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "math" "os" "path/filepath" @@ -24,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/grafana/grafana/pkg/infra/log/logtest" "github.com/grafana/grafana/pkg/infra/tracing" authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -766,6 +768,7 @@ func setupBleveBackend(t *testing.T, fileThreshold int, cacheTTL time.Duration, Root: dir, FileThreshold: int64(fileThreshold), IndexCacheTTL: cacheTTL, + Logger: slog.New(logtest.NewNopHandler(t)), }, tracing.NewNoopTracerService(), featuremgmt.WithFeatures(), metrics) require.NoError(t, err) require.NotNil(t, backend) @@ -773,173 +776,83 @@ func setupBleveBackend(t *testing.T, fileThreshold int, cacheTTL time.Duration, return backend, reg } -func TestBleveInMemoryIndexExpiration(t *testing.T) { - backend, reg := setupBleveBackend(t, 5, time.Nanosecond, "") - +func TestBuildIndexExpiration(t *testing.T) { ns := resource.NamespacedResource{ Namespace: "test", Group: "group", Resource: "resource", } - builtIndex, err := backend.BuildIndex(context.Background(), ns, 1 /* below FileThreshold */, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false, false) - require.NoError(t, err) + t.Run("memory based indexes should expire", func(t *testing.T) { + backend, reg := setupBleveBackend(t, 5, time.Nanosecond, "") - // Wait for index expiration, which is 1ns - time.Sleep(10 * time.Millisecond) - idx, err := backend.GetIndex(context.Background(), ns) - require.NoError(t, err) - require.Nil(t, idx) + builtIndex, err := backend.BuildIndex(context.Background(), ns, 1 /* below FileThreshold */, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false, false) + require.NoError(t, err) - // Verify that builtIndex is now closed. - _, err = builtIndex.DocCount(context.Background(), "") - require.ErrorIs(t, err, bleve.ErrorIndexClosed) + // Wait for index expiration, which is 1ns + time.Sleep(10 * time.Millisecond) + idx, err := backend.GetIndex(context.Background(), ns) + require.NoError(t, err) + require.Nil(t, idx) - // Verify that there are no open indexes. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) + // Verify that builtIndex is now closed. + _, err = builtIndex.DocCount(context.Background(), "") + require.ErrorIs(t, err, bleve.ErrorIndexClosed) + + // Verify that there are no open indexes. + checkOpenIndexes(t, reg, 0, 0) + }) + + t.Run("file based indexes should NOT expire", func(t *testing.T) { + backend, reg := setupBleveBackend(t, 5, time.Nanosecond, "") + + // size=100 is above FileThreshold, this will be file-based index + builtIndex, err := backend.BuildIndex(context.Background(), ns, 100, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false, false) + require.NoError(t, err) + + // Wait for index expiration, which is 1ns + time.Sleep(10 * time.Millisecond) + idx, err := backend.GetIndex(context.Background(), ns) + require.NoError(t, err) + require.NotNil(t, idx) + + // Verify that builtIndex is still open. + cnt, err := builtIndex.DocCount(context.Background(), "") + require.NoError(t, err) + require.Equal(t, int64(1), cnt) + + checkOpenIndexes(t, reg, 0, 1) + }) } -func TestBleveFileIndexExpiration(t *testing.T) { - backend, reg := setupBleveBackend(t, 5, time.Nanosecond, "") - +func TestCloseAllIndexes(t *testing.T) { ns := resource.NamespacedResource{ Namespace: "test", Group: "group", Resource: "resource", } - - // size=100 is above FileThreshold, this will be file-based index - builtIndex, err := backend.BuildIndex(context.Background(), ns, 100, 100, nil, "test", indexTestDocs(ns, 1, 100), nil, false, false) - require.NoError(t, err) - - // Wait for index expiration, which is 1ns - time.Sleep(10 * time.Millisecond) - idx, err := backend.GetIndex(context.Background(), ns) - require.NoError(t, err) - require.NotNil(t, idx) - - // Verify that builtIndex is still open. - cnt, err := builtIndex.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) -} - -func TestFileIndexIsReusedOnSameSizeAndRVLessThanIndexRV(t *testing.T) { - ns := resource.NamespacedResource{ - Namespace: "test", + ns2 := resource.NamespacedResource{ + Namespace: "test2", Group: "group", Resource: "resource", } tmpDir := t.TempDir() - - backend1, reg1 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) + backend1, reg := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) _, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, false) require.NoError(t, err) + _, err = backend1.BuildIndex(context.Background(), ns2, 1 /* memory based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, false) + require.NoError(t, err) - // Verify one open index. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - + // Verify two open indexes. + checkOpenIndexes(t, reg, 1, 1) backend1.CloseAllIndexes() // Verify that there are no open indexes after CloseAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // We open new backend using same directory, and run indexing with same size (10) and RV (100). This should reuse existing index, and skip indexing. - backend2, reg2 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 1000, 100), nil, false, false) - require.NoError(t, err) - - // Verify that we're reusing existing index and there is only 10 documents in it, not 1000. - cnt, err := idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(10), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend2.CloseAllIndexes() - // Verify that there are no open indexes after closeAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // We repeat with backend3 and RV 99. This should also reuse existing index and skip indexing - backend3, reg3 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend3.BuildIndex(context.Background(), ns, 10 /* file based */, 99, nil, "test", indexTestDocs(ns, 1000, 99), nil, false, false) - require.NoError(t, err) - - // Verify that we're reusing existing index and there is only 10 documents in it, not 1000. - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(10), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg3, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend3.CloseAllIndexes() - - require.NoError(t, testutil.GatherAndCompare(reg3, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // again now RV > 100. Should NOT reuse index - backend4, reg4 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend4.BuildIndex(context.Background(), ns, 10 /* file based */, 101, nil, "test", indexTestDocs(ns, 1000, 100), nil, false, false) - require.NoError(t, err) - - // Verify that we're NOT existing index and there is only 1000 documents in it, not 10. - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1000), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg4, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend4.CloseAllIndexes() + checkOpenIndexes(t, reg, 0, 0) } -func TestFileIndexIsIgnoredIfRebuildFlagIsTrueWithoutSearchAfterWrite(t *testing.T) { +func TestBuildIndex(t *testing.T) { ns := resource.NamespacedResource{ Namespace: "test", Group: "group", @@ -948,335 +861,93 @@ func TestFileIndexIsIgnoredIfRebuildFlagIsTrueWithoutSearchAfterWrite(t *testing tmpDir := t.TempDir() - backend1, reg1 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - _, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, true, false) - require.NoError(t, err) + type RV string + const ( + RVLessThan RV = "less" + RVBiggerThan RV = "more" + RVSame RV = "same" + ) + for _, searchAfterWrite := range []bool{false, true} { + for _, rebuild := range []bool{false, true} { + for _, sameSize := range []bool{false, true} { + for _, documentRV := range []RV{RVLessThan, RVSame, RVBiggerThan} { + shouldRebuild := false + if rebuild || !sameSize || (!searchAfterWrite && documentRV == RVBiggerThan) { + shouldRebuild = true + } - // Verify one open index. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) + testName := "" + if shouldRebuild { + testName += "should NOT reuse index " + } else { + testName += "should reuse index " + } - backend1.CloseAllIndexes() + if sameSize { + testName += "on same size " + } else { + testName += "on different size " + } - // Verify that there are no open indexes after CloseAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) + switch documentRV { + case RVLessThan: + testName += "and documentRV < indexRV " + case RVBiggerThan: + testName += "and documentRV > indexRV " + case RVSame: + testName += "and documentRV = indexRV " + } - // We open new backend using same directory, and run indexing with same size (10) and RV (100). This should NOT - // reuse existing index, due to the rebuild flag being true - backend2, reg2 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 1000, 100), nil, true, false) - require.NoError(t, err) + if rebuild { + testName += "when rebuild is true " + } else { + testName += "when rebuild is false " + } - // Verify that we've re-built the index. There should be 1000 documents, not 10. - cnt, err := idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1000), cnt) + if searchAfterWrite { + testName += "and searchAfterWrite is true" + } else { + testName += "and searchAfterWrite is false" + } - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) + t.Run(testName, func(t *testing.T) { + var size int64 = 10 + var rv int64 = 100 + backend1, _ := createBleveBackendAndIndex(t, tmpDir, ns, size, rv, 10, rebuild, searchAfterWrite) + backend1.CloseAllIndexes() - backend2.CloseAllIndexes() - // Verify that there are no open indexes after closeAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) + if !sameSize { + size = 11 + } + switch documentRV { + case RVBiggerThan: + rv = 101 + case RVLessThan: + rv = 99 + case RVSame: + } + backend2, idx := createBleveBackendAndIndex(t, tmpDir, ns, size, rv, 1000, rebuild, searchAfterWrite) - // We repeat with backend3 and RV 99. This should also NOT reuse existing index - backend3, reg3 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend3.BuildIndex(context.Background(), ns, 10 /* file based */, 99, nil, "test", indexTestDocs(ns, 1001, 99), nil, true, false) - require.NoError(t, err) - - // Verify that we've re-built the index. There should be 1001 documents - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1001), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg3, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend3.CloseAllIndexes() - - // again now RV > 100. Should still rebuild the index due to the rebuild flag - backend4, reg4 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend4.BuildIndex(context.Background(), ns, 10 /* file based */, 101, nil, "test", indexTestDocs(ns, 1002, 100), nil, true, false) - require.NoError(t, err) - - // Verify that we've re-built the index. There should be 1002 documents - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1002), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg4, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend4.CloseAllIndexes() + cnt, err := idx.DocCount(context.Background(), "") + require.NoError(t, err) + if shouldRebuild { + require.Equal(t, int64(1000), cnt) + } else { + require.Equal(t, int64(10), cnt) + } + backend2.CloseAllIndexes() + }) + } + } + } + } } -func TestFileIndexIsIgnoredIfRebuildFlagIsTrueWithSearchAfterWrite(t *testing.T) { - ns := resource.NamespacedResource{ - Namespace: "test", - Group: "group", - Resource: "resource", - } - - tmpDir := t.TempDir() - - backend1, reg1 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - _, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, true, true) +func createBleveBackendAndIndex(t *testing.T, tmpDir string, ns resource.NamespacedResource, size, rv int64, docCount int, rebuild, searchAfterWrite bool) (*bleveBackend, resource.ResourceIndex) { + backend, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) + idx, err := backend.BuildIndex(context.Background(), ns, size /* file based */, rv, nil, "test", indexTestDocs(ns, docCount, rv), nil, rebuild, searchAfterWrite) require.NoError(t, err) - - // Verify one open index. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend1.CloseAllIndexes() - - // Verify that there are no open indexes after CloseAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // We open new backend using same directory, and run indexing with same size (10) and RV (100). This should NOT - // reuse existing index, due to the rebuild flag being true - backend2, reg2 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 1000, 100), nil, true, true) - require.NoError(t, err) - - // Verify that we've re-built the index. There should be 1000 documents, not 10. - cnt, err := idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1000), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend2.CloseAllIndexes() - // Verify that there are no open indexes after closeAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // We repeat with backend3 and RV 99. This should also NOT reuse existing index - backend3, reg3 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend3.BuildIndex(context.Background(), ns, 10 /* file based */, 99, nil, "test", indexTestDocs(ns, 1001, 99), nil, true, true) - require.NoError(t, err) - - // Verify that we've re-built the index. There should be 1001 documents - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1001), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg3, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend3.CloseAllIndexes() - - // again now RV > 100. Should still rebuild the index due to the rebuild flag - backend4, reg4 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend4.BuildIndex(context.Background(), ns, 10 /* file based */, 101, nil, "test", indexTestDocs(ns, 1002, 100), nil, true, true) - require.NoError(t, err) - - // Verify that we've re-built the index. There should be 1002 documents - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(1002), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg4, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend4.CloseAllIndexes() -} - -func TestFileIndexIsReusedIfRVisPresentAndSearhAfterWriteIsEnabled(t *testing.T) { - ns := resource.NamespacedResource{ - Namespace: "test", - Group: "group", - Resource: "resource", - } - - tmpDir := t.TempDir() - - backend1, reg1 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - _, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, true) - require.NoError(t, err) - - // Verify one open index. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend1.CloseAllIndexes() - - // Verify that there are no open indexes after CloseAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg1, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // We open new backend using same directory, and run indexing with same size (10) and RV (100). This should reuse existing index, and skip indexing. - backend2, reg2 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 1000, 100), nil, false, true) - require.NoError(t, err) - - // Verify that we're reusing existing index and there is only 10 documents in it, not 1000. - cnt, err := idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(10), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend2.CloseAllIndexes() - // Verify that there are no open indexes after closeAllIndexes call. - require.NoError(t, testutil.GatherAndCompare(reg2, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 0 - `), "index_server_open_indexes")) - - // We repeat with backend3 and RV 99. This should also reuse existing index and skip indexing - backend3, reg3 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend3.BuildIndex(context.Background(), ns, 10 /* file based */, 99, nil, "test", indexTestDocs(ns, 1000, 99), nil, false, true) - require.NoError(t, err) - - // Verify that we're reusing existing index and there is only 10 documents in it, not 1000. - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(10), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg3, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend3.CloseAllIndexes() - - // again now RV > 100. Should still reuse index - backend4, reg4 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err = backend4.BuildIndex(context.Background(), ns, 10 /* file based */, 101, nil, "test", indexTestDocs(ns, 1000, 100), nil, false, true) - require.NoError(t, err) - - // Verify that we're reusing existing index and there is only 10 documents in it, not 1000. - cnt, err = idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(10), cnt) - - require.NoError(t, testutil.GatherAndCompare(reg4, bytes.NewBufferString(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} 0 - index_server_open_indexes{index_storage="file"} 1 - `), "index_server_open_indexes")) - - backend4.CloseAllIndexes() -} - -func TestFileIndexIsNotReusedOnDifferentSize(t *testing.T) { - ns := resource.NamespacedResource{ - Namespace: "test", - Group: "group", - Resource: "resource", - } - - tmpDir := t.TempDir() - - backend1, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - _, err := backend1.BuildIndex(context.Background(), ns, 10, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, false) - require.NoError(t, err) - backend1.CloseAllIndexes() - - // We open new backend using same directory, but with different size. Index should be rebuilt. - backend2, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err := backend2.BuildIndex(context.Background(), ns, 100, 100, nil, "test", indexTestDocs(ns, 100, 100), nil, false, false) - require.NoError(t, err) - - // Verify that index has updated number of documents. - cnt, err := idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(100), cnt) -} - -func TestFileIndexIsNotReusedOnDifferentRV(t *testing.T) { - ns := resource.NamespacedResource{ - Namespace: "test", - Group: "group", - Resource: "resource", - } - - tmpDir := t.TempDir() - - backend1, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - _, err := backend1.BuildIndex(context.Background(), ns, 10, 100, nil, "test", indexTestDocs(ns, 10, 100), nil, false, false) - require.NoError(t, err) - backend1.CloseAllIndexes() - - // We open new backend using same directory, but with different RV. Index should be rebuilt. - backend2, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir) - idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 999999, nil, "test", indexTestDocs(ns, 100, 999999), nil, false, false) - require.NoError(t, err) - - // Verify that index has updated number of documents. - cnt, err := idx.DocCount(context.Background(), "") - require.NoError(t, err) - require.Equal(t, int64(100), cnt) + return backend, idx } func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) { @@ -1337,16 +1008,20 @@ func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(secondSize), cnt) - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` - # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. - # TYPE index_server_open_indexes gauge - index_server_open_indexes{index_storage="memory"} %d - index_server_open_indexes{index_storage="file"} %d - `, openInMemoryIndexes, 1-openInMemoryIndexes)), "index_server_open_indexes")) + checkOpenIndexes(t, reg, openInMemoryIndexes, 1-openInMemoryIndexes) }) } } +func checkOpenIndexes(t *testing.T, reg prometheus.Gatherer, memory, file int) { + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` + # HELP index_server_open_indexes Number of open indexes per storage type. An open index corresponds to single resource group. + # TYPE index_server_open_indexes gauge + index_server_open_indexes{index_storage="memory"} %d + index_server_open_indexes{index_storage="file"} %d + `, memory, file)), "index_server_open_indexes")) +} + func verifyDirEntriesCount(t *testing.T, dir string, count int) { ents, err := os.ReadDir(dir) if err != nil {