public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup of old search functionality (#110861)

Browse Source 
* Remove support for initMinSize.
Remove support for searchAfterWrite option, now it defaults to true.

* Remove reference to deprecated feature toggle.

* Remove feature toggle completely.

* Remove code related to indexing on watch events.

* Fix compilation error.

* Remove unused field.
This commit is contained in:
Peter Štibraný
2025-09-11 08:23:03 +00:00
committed by GitHub
parent ecf08ad7d5
commit 6fa6a5708a
23 changed files with 164 additions and 1054 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/testing/benchmark.go
+2 -90
View File
@@ -215,9 +215,9 @@ func runSearchBackendBenchmarkWriteThroughput(ctx context.Context, backend resou
// Build initial index
size := int64(10000) // force the index to be on disk
index, err := backend.BuildIndex(ctx, nr, size, 0, nil, "benchmark", func(index resource.ResourceIndex) (int64, error) {
index, err := backend.BuildIndex(ctx, nr, size, nil, "benchmark", func(index resource.ResourceIndex) (int64, error) {
return 0, nil
}, nil, false, false)
}, nil, false)
if err != nil {
return nil, fmt.Errorf("failed to initialize backend: %w", err)
}
@@ -334,91 +334,3 @@ func BenchmarkSearchBackend(tb testing.TB, backend resource.SearchBackend, opts
tb.Logf("P90 Latency: %v", result.P90Latency)
tb.Logf("P99 Latency: %v", result.P99Latency)
}
func BenchmarkIndexServer(tb testing.TB, ctx context.Context, backend resource.StorageBackend, searchBackend resource.SearchBackend, opts *BenchmarkOptions) {
events := make(chan *resource.IndexEvent, opts.NumResources)
groupsResources := make(map[string]string)
for g := 0; g < opts.NumGroups; g++ {
for r := 0; r < opts.NumResourceTypes; r++ {
groupsResources[fmt.Sprintf("group-%d", g)] = fmt.Sprintf("resource-%d", r)
}
}
server, err := resource.NewResourceServer(resource.ResourceServerOptions{
Backend: backend,
Search: resource.SearchOptions{
Backend: searchBackend,
IndexEventsChan: events,
Resources: &resource.TestDocumentBuilderSupplier{GroupsResources: groupsResources},
},
})
require.NoError(tb, err)
require.NotNil(tb, server)
// Initialize the backend
err = initializeBackend(ctx, backend, opts)
require.NoError(tb, err)
// Discard the latencies from the initial index build.
for i := 0; i < (opts.NumGroups * opts.NumResourceTypes * opts.NumNamespaces); i++ {
<-events
}
// Run the storage backend benchmark write throughput to create events
startTime := time.Now()
var result *BenchmarkResult
// Channel to signal when the benchmark goroutine completes
benchmarkDone := make(chan struct{})
go func() {
defer close(benchmarkDone)
result, err = runStorageBackendBenchmark(ctx, backend, opts)
require.NoError(tb, err)
}()
// Wait for all events to be processed
latencies := make([]float64, 0, opts.NumResources)
for i := 0; i < opts.NumResources; i++ {
evt := <-events
latencies = append(latencies, evt.Latency.Seconds())
}
totalDuration := time.Since(startTime)
<-benchmarkDone
// Calculate index latency percentiles
sort.Float64s(latencies)
var p50, p90, p99 float64
if len(latencies) > 0 {
p50 = latencies[len(latencies)*50/100]
p90 = latencies[len(latencies)*90/100]
p99 = latencies[len(latencies)*99/100]
}
// Report metrics if running a benchmark
if b, ok := tb.(*testing.B); ok {
b.ReportMetric(result.Throughput, "writes/sec")
b.ReportMetric(float64(result.P50Latency.Milliseconds()), "p50-latency-ms")
b.ReportMetric(float64(result.P90Latency.Milliseconds()), "p90-latency-ms")
b.ReportMetric(float64(result.P99Latency.Milliseconds()), "p99-latency-ms")
b.ReportMetric(p50, "p50-index-latency-s")
b.ReportMetric(p90, "p90-index-latency-s")
b.ReportMetric(p99, "p99-index-latency-s")
}
// Log results for better visibility
tb.Logf("Benchmark Configuration: Workers=%d, Resources=%d, Namespaces=%d, Groups=%d, Resource Types=%d",
opts.Concurrency, opts.NumResources, opts.NumNamespaces, opts.NumGroups, opts.NumResourceTypes)
tb.Logf("")
tb.Logf("Storage Benchmark Results:")
tb.Logf("Total Duration: %v", result.TotalDuration)
tb.Logf("Storage Write Count: %d", result.WriteCount)
tb.Logf("Storage Write Throughput: %.2f writes/sec", result.Throughput)
tb.Logf("P50 Write Latency: %v", result.P50Latency)
tb.Logf("P90 Write Latency: %v", result.P90Latency)
tb.Logf("P99 Write Latency: %v", result.P99Latency)
tb.Logf("")
tb.Logf("Index Latency Results:")
tb.Logf("Indexing Throughput: %.2f events/sec", float64(len(latencies))/totalDuration.Seconds())
tb.Logf("P50 Index Latency: %.3fs", p50)
tb.Logf("P90 Index Latency: %.3fs", p90)
tb.Logf("P99 Index Latency: %.3fs", p99)
}
pkg/storage/unified/testing/search_and_storage.go
-5
View File
@@ -94,7 +94,6 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
require.Greater(t, rv, int64(0))
}
})
ch := make(chan *resource.IndexEvent)
t.Run("Create a resource server with both backends", func(t *testing.T) {
// Create a resource server with both backends
@@ -108,7 +107,6 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
"test.grafana.app": "testresources",
},
},
IndexEventsChan: ch,
},
})
require.NoError(t, err)
@@ -192,9 +190,6 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
require.NoError(t, err)
require.NotNil(t, createResp)
require.Nil(t, createResp.Error)
ev := <-ch
require.NotNil(t, ev)
}
})
pkg/storage/unified/testing/search_backend.go
+6 -6
View File
@@ -64,7 +64,7 @@ func runTestSearchBackendBuildIndex(t *testing.T, backend resource.SearchBackend
require.Nil(t, index)
// Build the index
index, err = backend.BuildIndex(ctx, ns, 0, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
index, err = backend.BuildIndex(ctx, ns, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
// Write a test document
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
@@ -86,7 +86,7 @@ func runTestSearchBackendBuildIndex(t *testing.T, backend resource.SearchBackend
return 0, err
}
return 1, nil
}, nil, false, false)
}, nil, false)
require.NoError(t, err)
require.NotNil(t, index)
@@ -111,7 +111,7 @@ func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix
}
// Build initial index with some test documents
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
index, err := backend.BuildIndex(ctx, ns, 3, nil, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -152,7 +152,7 @@ func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix
})
require.NoError(t, err)
return int64(2), nil
}, nil, false, false)
}, nil, false)
require.NoError(t, err)
require.NotNil(t, index)
@@ -235,7 +235,7 @@ func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix
t.Run("Search by LibraryPanel reference", func(t *testing.T) {
// Build index with dashboards that have LibraryPanel references
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, "test", func(index resource.ResourceIndex) (int64, error) {
index, err := backend.BuildIndex(ctx, ns, 3, nil, "test", func(index resource.ResourceIndex) (int64, error) {
err := index.BulkIndex(&resource.BulkIndexRequest{
Items: []*resource.BulkIndexItem{
{
@@ -294,7 +294,7 @@ func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix
})
require.NoError(t, err)
return int64(3), nil
}, nil, false, false)
}, nil, false)
require.NoError(t, err)
require.NotNil(t, index)