public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve indexing observability (#108901)

Browse Source 
* Add details to tracing spans when creating index.

* Log reason for building index.

* Log reason for building index.

* Remove initialization of labels to avoid unnecessary metrics.

* Track succcessful, failed and skiped index builds. Track index build time for individual index, not all indexes.

* Revert removal of labels initialization.
This commit is contained in:
Peter Štibraný
2025-07-30 16:34:15 +02:00
committed by GitHub
parent 98e37f2ca9
commit c28b2215e0
9 changed files with 108 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/resource/bleve_index_metrics.go
+22 -8
View File
@@ -9,11 +9,14 @@ import (
)
type BleveIndexMetrics struct {
IndexLatency *prometheus.HistogramVec
IndexSize prometheus.Gauge
IndexedKinds *prometheus.GaugeVec
IndexCreationTime *prometheus.HistogramVec
OpenIndexes *prometheus.GaugeVec
IndexLatency *prometheus.HistogramVec
IndexSize prometheus.Gauge
IndexedKinds *prometheus.GaugeVec
IndexCreationTime *prometheus.HistogramVec
OpenIndexes *prometheus.GaugeVec
IndexBuilds *prometheus.CounterVec
IndexBuildFailures prometheus.Counter
IndexBuildSkipped prometheus.Counter
}
var IndexCreationBuckets = []float64{1, 5, 10, 25, 50, 75, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}
@@ -37,8 +40,8 @@ func ProvideIndexMetrics(reg prometheus.Registerer) *BleveIndexMetrics {
Help: "Number of indexed documents by kind",
}, []string{"kind"}),
IndexCreationTime: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
Name: "index_server_index_creation_time_seconds",
Help: "Time (in seconds) it takes until index is created",
Name: "index_server_index_build_time_seconds",
Help: "Time it takes to successfully build an index. Failed or skipped builds are not counted.",
Buckets: IndexCreationBuckets,
NativeHistogramBucketFactor: 1.1, // enable native histograms
NativeHistogramMaxBucketNumber: 160,
@@ -48,11 +51,22 @@ func ProvideIndexMetrics(reg prometheus.Registerer) *BleveIndexMetrics {
Name: "index_server_open_indexes",
Help: "Number of open indexes per storage type. An open index corresponds to single resource group.",
}, []string{"index_storage"}), // index_storage is either "file" or "memory"
IndexBuilds: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
Name: "index_server_index_build_total",
Help: "Number of times index build was attempted due to specific reason",
}, []string{"reason"}),
IndexBuildFailures: promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "index_server_index_build_failures_total",
Help: "Number of times index build failed",
}),
IndexBuildSkipped: promauto.With(reg).NewCounter(prometheus.CounterOpts{
Name: "index_server_index_build_skipped_total",
Help: "Number of times index build has been skipped due to existing valid index being found on disk",
}),
}
// Initialize labels.
m.OpenIndexes.WithLabelValues("file").Set(0)
m.OpenIndexes.WithLabelValues("memory").Set(0)
return m
}
pkg/storage/unified/resource/bulk.go
+1 -1
View File
@@ -244,7 +244,7 @@ func (s *server) BulkProcess(stream resourcepb.BulkStore_BulkProcessServer) erro
Namespace: summary.Namespace,
Group: summary.Group,
Resource: summary.Resource,
}, summary.Count, summary.ResourceVersion)
}, summary.Count, summary.ResourceVersion, "rebuildAfterBatchLoad")
if err != nil {
s.log.Warn("error building search index after batch load", "err", err)
rsp.Error = &resourcepb.ErrorResult{
pkg/storage/unified/resource/search.go
+34 -23
View File
@@ -90,7 +90,7 @@ type SearchBackend interface {
// Depending on the size, the backend may choose different options (eg: memory vs disk).
// The last known resource version can be used to detect that nothing has changed, and existing on-disk index can be reused.
// The builder will write all documents before returning.
BuildIndex(ctx context.Context, key NamespacedResource, size int64, resourceVersion int64, nonStandardFields SearchableDocumentFields, builder func(index ResourceIndex) (int64, error)) (ResourceIndex, error)
BuildIndex(ctx context.Context, key NamespacedResource, size int64, resourceVersion int64, nonStandardFields SearchableDocumentFields, indexBuildReason string, builder func(index ResourceIndex) (int64, error)) (ResourceIndex, error)
// TotalDocs returns the total number of documents across all indexes.
TotalDocs() int64
@@ -196,7 +196,7 @@ func (s *searchSupport) ListManagedObjects(ctx context.Context, req *resourcepb.
Namespace: req.Namespace,
Group: info.Group,
Resource: info.Resource,
})
}, "listManagedObjects")
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
@@ -237,7 +237,7 @@ func (s *searchSupport) CountManagedObjects(ctx context.Context, req *resourcepb
Namespace: req.Namespace,
Group: info.Group,
Resource: info.Resource,
})
}, "countManagedObjects")
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
@@ -282,7 +282,7 @@ func (s *searchSupport) Search(ctx context.Context, req *resourcepb.ResourceSear
Namespace: req.Options.Key.Namespace,
Resource: req.Options.Key.Resource,
}
idx, err := s.getOrCreateIndex(ctx, nsr)
idx, err := s.getOrCreateIndex(ctx, nsr, "search")
if err != nil {
return &resourcepb.ResourceSearchResponse{
Error: AsErrorResult(err),
@@ -294,7 +294,7 @@ func (s *searchSupport) Search(ctx context.Context, req *resourcepb.ResourceSear
for i, f := range req.Federated {
nsr.Group = f.Group
nsr.Resource = f.Resource
federate[i], err = s.getOrCreateIndex(ctx, nsr)
federate[i], err = s.getOrCreateIndex(ctx, nsr, "federatedSearch")
if err != nil {
return &resourcepb.ResourceSearchResponse{
Error: AsErrorResult(err),
@@ -323,7 +323,7 @@ func (s *searchSupport) GetStats(ctx context.Context, req *resourcepb.ResourceSt
Namespace: req.Namespace,
Group: parts[0],
Resource: parts[1],
})
}, "getStats")
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
@@ -367,7 +367,7 @@ func (s *searchSupport) GetStats(ctx context.Context, req *resourcepb.ResourceSt
Namespace: req.Namespace,
Group: stat.Group,
Resource: stat.Resource,
})
}, "getStats")
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
@@ -449,8 +449,12 @@ func (s *searchSupport) buildIndexes(ctx context.Context, rebuild bool) (int, er
return err
}
s.log.Debug("building index", "namespace", info.Namespace, "group", info.Group, "resource", info.Resource)
_, _, err := s.build(ctx, info.NamespacedResource, info.Count, info.ResourceVersion)
s.log.Debug("building index", "namespace", info.Namespace, "group", info.Group, "resource", info.Resource, "rebuild", rebuild)
reason := "init"
if rebuild {
reason = "rebuild"
}
_, _, err := s.build(ctx, info.NamespacedResource, info.Count, info.ResourceVersion, reason)
return err
})
}
@@ -504,9 +508,6 @@ func (s *searchSupport) init(ctx context.Context) error {
end := time.Now().Unix()
s.log.Info("search index initialized", "duration_secs", end-start, "total_docs", s.search.TotalDocs())
if s.indexMetrics != nil {
s.indexMetrics.IndexCreationTime.WithLabelValues().Observe(float64(end - start))
}
return nil
}
@@ -537,7 +538,7 @@ func (s *searchSupport) dispatchEvent(ctx context.Context, evt *WrittenEvent) {
Group: evt.Key.Group,
Resource: evt.Key.Resource,
}
index, err := s.getOrCreateIndex(ctx, nsr)
index, err := s.getOrCreateIndex(ctx, nsr, "dispatchEvent")
if err != nil {
s.log.Warn("error getting index for watch event", "error", err)
span.RecordError(err)
@@ -622,15 +623,10 @@ func (s *searchSupport) rebuildDashboardIndexes(ctx context.Context) error {
"duration", duration,
"rebuilt_indexes", totalBatchesIndexed,
"total_docs", s.search.TotalDocs())
if s.indexMetrics != nil {
s.indexMetrics.IndexCreationTime.WithLabelValues().Observe(duration.Seconds())
}
return nil
}
func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedResource) (ResourceIndex, error) {
func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedResource, reason string) (ResourceIndex, error) {
if s == nil || s.search == nil {
return nil, fmt.Errorf("search is not configured properly (missing unifiedStorageSearch feature toggle?)")
}
@@ -672,7 +668,7 @@ func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedReso
}
}
idx, _, err = s.build(ctx, key, size, rv)
idx, _, err = s.build(ctx, key, size, rv, reason)
if err != nil {
return nil, fmt.Errorf("error building search index, %w", err)
}
@@ -693,10 +689,18 @@ func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedReso
}
}
func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size int64, rv int64) (ResourceIndex, int64, error) {
func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size int64, rv int64, indexBuildReason string) (ResourceIndex, int64, error) {
ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Build")
defer span.End()
span.SetAttributes(
attribute.String("namespace", nsr.Namespace),
attribute.String("group", nsr.Group),
attribute.String("resource", nsr.Resource),
attribute.Int64("size", size),
attribute.Int64("rv", rv),
)
logger := s.log.With("namespace", nsr.Namespace, "group", nsr.Group, "resource", nsr.Resource)
builder, err := s.builders.get(ctx, nsr)
@@ -705,7 +709,10 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
}
fields := s.builders.GetFields(nsr)
index, err := s.search.BuildIndex(ctx, nsr, size, rv, fields, func(index ResourceIndex) (int64, error) {
index, err := s.search.BuildIndex(ctx, nsr, size, rv, fields, indexBuildReason, func(index ResourceIndex) (int64, error) {
span := trace.SpanFromContext(ctx)
span.AddEvent("building index", trace.WithAttributes(attribute.Int64("size", size), attribute.Int64("rv", rv), attribute.String("reason", indexBuildReason)))
rv, err = s.storage.ListIterator(ctx, &resourcepb.ListRequest{
Limit: 1000000000000, // big number
Options: &resourcepb.ListOptions{
@@ -734,9 +741,11 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
Name: iter.Name(),
}
span.AddEvent("building document", trace.WithAttributes(attribute.String("name", iter.Name())))
// Convert it to an indexable document
doc, err := builder.BuildDocument(ctx, key, iter.ResourceVersion(), iter.Value())
if err != nil {
span.RecordError(err)
logger.Error("error building search document", "key", SearchID(key), "err", err)
continue
}
@@ -749,6 +758,7 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
// When we reach the batch size, perform bulk index and reset the batch.
if len(items) >= maxBatchSize {
span.AddEvent("bulk indexing", trace.WithAttributes(attribute.Int("count", len(items))))
if err = index.BulkIndex(&BulkIndexRequest{
Items: items,
}); err != nil {
@@ -762,6 +772,7 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
// Index any remaining items in the final batch.
if len(items) > 0 {
span.AddEvent("bulk indexing", trace.WithAttributes(attribute.Int("count", len(items))))
if err = index.BulkIndex(&BulkIndexRequest{
Items: items,
}); err != nil {
@@ -799,7 +810,7 @@ func (s *searchSupport) buildEmptyIndex(ctx context.Context, nsr NamespacedResou
s.log.Debug("Building empty index", "namespace", nsr.Namespace, "group", nsr.Group, "resource", nsr.Resource, "rv", rv)
// Build an empty index by passing a builder function that doesn't add any documents
return s.search.BuildIndex(ctx, nsr, 0, rv, fields, func(index ResourceIndex) (int64, error) {
return s.search.BuildIndex(ctx, nsr, 0, rv, fields, "empty", func(index ResourceIndex) (int64, error) {
// Return the resource version without adding any documents to the index
return rv, nil
})
pkg/storage/unified/resource/search_test.go
+5 -5
View File
@@ -121,7 +121,7 @@ func (m *mockSearchBackend) GetIndex(ctx context.Context, key NamespacedResource
return nil, nil
}
func (m *mockSearchBackend) BuildIndex(ctx context.Context, key NamespacedResource, size int64, resourceVersion int64, fields SearchableDocumentFields, builder func(index ResourceIndex) (int64, error)) (ResourceIndex, error) {
func (m *mockSearchBackend) BuildIndex(ctx context.Context, key NamespacedResource, size int64, resourceVersion int64, fields SearchableDocumentFields, reason string, builder func(index ResourceIndex) (int64, error)) (ResourceIndex, error) {
index := &MockResourceIndex{}
index.On("BulkIndex", mock.Anything).Return(nil).Maybe()
index.On("DocCount", mock.Anything, mock.Anything).Return(int64(0), nil).Maybe()
@@ -317,7 +317,7 @@ func TestSearchGetOrCreateIndex(t *testing.T) {
go func() {
defer wg.Done()
<-start
_, _ = support.getOrCreateIndex(context.Background(), NamespacedResource{Namespace: "ns", Group: "group", Resource: "resource"})
_, _ = support.getOrCreateIndex(context.Background(), NamespacedResource{Namespace: "ns", Group: "group", Resource: "resource"}, "test")
}()
}
@@ -365,7 +365,7 @@ func TestSearchGetOrCreateIndexWithCancellation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
defer cancel()
_, err = support.getOrCreateIndex(ctx, NamespacedResource{Namespace: "ns", Group: "group", Resource: "resource"})
_, err = support.getOrCreateIndex(ctx, NamespacedResource{Namespace: "ns", Group: "group", Resource: "resource"}, "test")
// Make sure we get context deadline error
require.ErrorIs(t, err, context.DeadlineExceeded)
@@ -380,9 +380,9 @@ type slowSearchBackend struct {
wg sync.WaitGroup
}
func (m *slowSearchBackend) BuildIndex(ctx context.Context, key NamespacedResource, size int64, resourceVersion int64, fields SearchableDocumentFields, builder func(index ResourceIndex) (int64, error)) (ResourceIndex, error) {
func (m *slowSearchBackend) BuildIndex(ctx context.Context, key NamespacedResource, size int64, resourceVersion int64, fields SearchableDocumentFields, reason string, builder func(index ResourceIndex) (int64, error)) (ResourceIndex, error) {
m.wg.Add(1)
defer m.wg.Done()
time.Sleep(1 * time.Second)
return m.mockSearchBackend.BuildIndex(ctx, key, size, resourceVersion, fields, builder)
return m.mockSearchBackend.BuildIndex(ctx, key, size, resourceVersion, fields, reason, builder)
}
pkg/storage/unified/search/bleve.go
+27 -1
View File
@@ -23,6 +23,7 @@ import (
"github.com/blevesearch/bleve/v2/search/query"
bleveSearch "github.com/blevesearch/bleve/v2/search/searcher"
index "github.com/blevesearch/bleve_index_api"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"k8s.io/apimachinery/pkg/selection"
@@ -197,11 +198,21 @@ func (b *bleveBackend) BuildIndex(
size int64,
resourceVersion int64,
fields resource.SearchableDocumentFields,
indexBuildReason string,
builder func(index resource.ResourceIndex) (int64, error),
) (resource.ResourceIndex, error) {
_, span := b.tracer.Start(ctx, tracingPrexfixBleve+"BuildIndex")
defer span.End()
span.SetAttributes(
attribute.String("namespace", key.Namespace),
attribute.String("group", key.Group),
attribute.String("resource", key.Resource),
attribute.Int64("size", size),
attribute.Int64("rv", resourceVersion),
attribute.String("reason", indexBuildReason),
)
mapper, err := GetBleveMappings(fields)
if err != nil {
return nil, err
@@ -214,7 +225,7 @@ func (b *bleveBackend) BuildIndex(
return nil, err
}
logWithDetails := b.log.With("namespace", key.Namespace, "group", key.Group, "resource", key.Resource, "size", size, "rv", resourceVersion)
logWithDetails := b.log.With("namespace", key.Namespace, "group", key.Group, "resource", key.Resource, "size", size, "rv", resourceVersion, "reason", indexBuildReason)
// Close the newly created/opened index by default.
closeIndex := true
@@ -306,14 +317,29 @@ func (b *bleveBackend) BuildIndex(
}
if build {
if b.indexMetrics != nil {
b.indexMetrics.IndexBuilds.WithLabelValues(indexBuildReason).Inc()
}
start := time.Now()
_, err = builder(idx)
if err != nil {
logWithDetails.Error("Failed to build index", "err", err)
if b.indexMetrics != nil {
b.indexMetrics.IndexBuildFailures.Inc()
}
return nil, fmt.Errorf("failed to build index: %w", err)
}
elapsed := time.Since(start)
logWithDetails.Info("Finished building index", "elapsed", elapsed)
if b.indexMetrics != nil {
b.indexMetrics.IndexCreationTime.WithLabelValues().Observe(elapsed.Seconds())
}
} else {
logWithDetails.Info("Skipping index build, using existing index")
if b.indexMetrics != nil {
b.indexMetrics.IndexBuildSkipped.Inc()
}
}
// Set expiration after building the index. Only expire in-memory indexes.
pkg/storage/unified/search/bleve_search_test.go
+1 -1
View File
@@ -553,7 +553,7 @@ func newTestDashboardsIndex(t TB, threshold int64, size int64, batchSize int64,
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
}, size, rv, info.Fields, writer)
}, size, rv, info.Fields, "test", writer)
require.NoError(t, err)
return index, tmpdir
pkg/storage/unified/search/bleve_test.go
+14 -14
View File
@@ -71,7 +71,7 @@ func TestBleveBackend(t *testing.T) {
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
}, 2, rv, info.Fields, func(index resource.ResourceIndex) (int64, error) {
}, 2, rv, info.Fields, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -352,7 +352,7 @@ func TestBleveBackend(t *testing.T) {
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
}, 2, rv, fields, func(index resource.ResourceIndex) (int64, error) {
}, 2, rv, fields, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -766,7 +766,7 @@ func TestBleveInMemoryIndexExpiration(t *testing.T) {
Resource: "resource",
}
builtIndex, err := backend.BuildIndex(context.Background(), ns, 1 /* below FileThreshold */, 100, nil, indexTestDocs(ns, 1))
builtIndex, err := backend.BuildIndex(context.Background(), ns, 1 /* below FileThreshold */, 100, nil, "test", indexTestDocs(ns, 1))
require.NoError(t, err)
// Wait for index expiration, which is 1ns
@@ -798,7 +798,7 @@ func TestBleveFileIndexExpiration(t *testing.T) {
}
// size=100 is above FileThreshold, this will be file-based index
builtIndex, err := backend.BuildIndex(context.Background(), ns, 100, 100, nil, indexTestDocs(ns, 1))
builtIndex, err := backend.BuildIndex(context.Background(), ns, 100, 100, nil, "test", indexTestDocs(ns, 1))
require.NoError(t, err)
// Wait for index expiration, which is 1ns
@@ -830,7 +830,7 @@ func TestFileIndexIsReusedOnSameSizeAndRV(t *testing.T) {
tmpDir := t.TempDir()
backend1, reg1 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
_, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, indexTestDocs(ns, 10))
_, err := backend1.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 10))
require.NoError(t, err)
// Verify one open index.
@@ -853,7 +853,7 @@ func TestFileIndexIsReusedOnSameSizeAndRV(t *testing.T) {
// We open new backend using same directory, and run indexing with same size (10) and RV (100). This should reuse existing index, and skip indexing.
backend2, reg2 := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, indexTestDocs(ns, 1000))
idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 100, nil, "test", indexTestDocs(ns, 1000))
require.NoError(t, err)
// Verify that we're reusing existing index and there is only 10 documents in it, not 1000.
@@ -879,13 +879,13 @@ func TestFileIndexIsNotReusedOnDifferentSize(t *testing.T) {
tmpDir := t.TempDir()
backend1, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
_, err := backend1.BuildIndex(context.Background(), ns, 10, 100, nil, indexTestDocs(ns, 10))
_, err := backend1.BuildIndex(context.Background(), ns, 10, 100, nil, "test", indexTestDocs(ns, 10))
require.NoError(t, err)
backend1.closeAllIndexes()
// We open new backend using same directory, but with different size. Index should be rebuilt.
backend2, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
idx, err := backend2.BuildIndex(context.Background(), ns, 100, 100, nil, indexTestDocs(ns, 100))
idx, err := backend2.BuildIndex(context.Background(), ns, 100, 100, nil, "test", indexTestDocs(ns, 100))
require.NoError(t, err)
// Verify that index has updated number of documents.
@@ -904,13 +904,13 @@ func TestFileIndexIsNotReusedOnDifferentRV(t *testing.T) {
tmpDir := t.TempDir()
backend1, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
_, err := backend1.BuildIndex(context.Background(), ns, 10, 100, nil, indexTestDocs(ns, 10))
_, err := backend1.BuildIndex(context.Background(), ns, 10, 100, nil, "test", indexTestDocs(ns, 10))
require.NoError(t, err)
backend1.closeAllIndexes()
// We open new backend using same directory, but with different RV. Index should be rebuilt.
backend2, _ := setupBleveBackend(t, 5, time.Nanosecond, tmpDir)
idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 999999, nil, indexTestDocs(ns, 100))
idx, err := backend2.BuildIndex(context.Background(), ns, 10 /* file based */, 999999, nil, "test", indexTestDocs(ns, 100))
require.NoError(t, err)
// Verify that index has updated number of documents.
@@ -942,7 +942,7 @@ func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) {
if testCase.firstInMemory {
firstSize = 1
}
firstIndex, err := backend.BuildIndex(context.Background(), ns, int64(firstSize), 100, nil, indexTestDocs(ns, firstSize))
firstIndex, err := backend.BuildIndex(context.Background(), ns, int64(firstSize), 100, nil, "test", indexTestDocs(ns, firstSize))
require.NoError(t, err)
openInMemoryIndexes := 0
@@ -952,7 +952,7 @@ func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) {
secondSize = 1
openInMemoryIndexes = 1
}
secondIndex, err := backend.BuildIndex(context.Background(), ns, int64(secondSize), 100, nil, indexTestDocs(ns, secondSize))
secondIndex, err := backend.BuildIndex(context.Background(), ns, int64(secondSize), 100, nil, "test", indexTestDocs(ns, secondSize))
require.NoError(t, err)
// Verify that first and second index are different, and first one is now closed.
@@ -1050,12 +1050,12 @@ func testBleveIndexWithFailures(t *testing.T, fileBased bool) {
// size=100 is above FileThreshold (5), make it a file-based index.
size = 100
}
_, err := backend.BuildIndex(context.Background(), ns, size, 100, nil, func(index resource.ResourceIndex) (int64, error) {
_, err := backend.BuildIndex(context.Background(), ns, size, 100, nil, "test", func(index resource.ResourceIndex) (int64, error) {
return 0, fmt.Errorf("fail")
})
require.Error(t, err)
// Even though previous build of the index failed, new building of the index should work.
_, err = backend.BuildIndex(context.Background(), ns, size, 100, nil, indexTestDocs(ns, int(size)))
_, err = backend.BuildIndex(context.Background(), ns, size, 100, nil, "test", indexTestDocs(ns, int(size)))
require.NoError(t, err)
}
pkg/storage/unified/testing/benchmark.go
+1 -1
View File
@@ -215,7 +215,7 @@ func runSearchBackendBenchmarkWriteThroughput(ctx context.Context, backend resou
// Build initial index
size := int64(10000) // force the index to be on disk
index, err := backend.BuildIndex(ctx, nr, size, 0, nil, func(index resource.ResourceIndex) (int64, error) {
index, err := backend.BuildIndex(ctx, nr, size, 0, nil, "benchmark", func(index resource.ResourceIndex) (int64, error) {
return 0, nil
})
if err != nil {
pkg/storage/unified/testing/search_backend.go
+3 -3
View File
@@ -64,7 +64,7 @@ func runTestSearchBackendBuildIndex(t *testing.T, backend resource.SearchBackend
require.Nil(t, index)
// Build the index
index, err = backend.BuildIndex(ctx, ns, 0, 0, nil, func(index resource.ResourceIndex) (int64, error) {
index, err = backend.BuildIndex(ctx, ns, 0, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
// Write a test document
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
@@ -111,7 +111,7 @@ func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix
}
// Build initial index with some test documents
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, func(index resource.ResourceIndex) (int64, error) {
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -235,7 +235,7 @@ func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix
t.Run("Search by LibraryPanel reference", func(t *testing.T) {
// Build index with dashboards that have LibraryPanel references
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, func(index resource.ResourceIndex) (int64, error) {
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{