public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve search index eviction (#111542)

Browse Source 
* Modify index eviction mechanism such that unowned indexes are also evicted.

* Propagate OwnsIndex function to bleve backend

Fix tests.

Stop eviction goroutine when stopping backend.

Make linter happy.

Make sure we stop backend created by tests.

Review suggestion.

Removed newline.
This commit is contained in:
Peter Štibraný
2025-09-24 16:54:35 +02:00
committed by GitHub
parent a4dabc31ed
commit 357aa7d314
13 changed files with 266 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/resource/search.go
+13 -35
View File
@@ -4,7 +4,6 @@ import (
"cmp"
"context"
"fmt"
"hash/fnv"
"log/slog"
"slices"
"strings"
@@ -20,7 +19,6 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/grafana/authlib/types"
"github.com/grafana/dskit/ring"
dashboardv1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1"
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
@@ -130,8 +128,7 @@ type searchSupport struct {
initWorkers int
initMinSize int
ring *ring.Ring
ringLifecycler *ring.BasicLifecycler
ownsIndexFn func(key NamespacedResource) (bool, error)
buildIndex singleflight.Group
@@ -144,7 +141,7 @@ var (
_ resourcepb.ManagedObjectIndexServer = (*searchSupport)(nil)
)
func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.AccessClient, blob BlobSupport, tracer trace.Tracer, indexMetrics *BleveIndexMetrics, ring *ring.Ring, ringLifecycler *ring.BasicLifecycler) (support *searchSupport, err error) {
func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.AccessClient, blob BlobSupport, tracer trace.Tracer, indexMetrics *BleveIndexMetrics, ownsIndexFn func(key NamespacedResource) (bool, error)) (support *searchSupport, err error) {
// No backend search support
if opts.Backend == nil {
return nil, nil
@@ -157,6 +154,12 @@ func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.A
opts.WorkerThreads = 1
}
if ownsIndexFn == nil {
ownsIndexFn = func(key NamespacedResource) (bool, error) {
return true, nil
}
}
support = &searchSupport{
access: access,
tracer: tracer,
@@ -167,8 +170,7 @@ func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.A
initMinSize: opts.InitMinCount,
indexMetrics: indexMetrics,
rebuildInterval: opts.RebuildInterval,
ring: ring,
ringLifecycler: ringLifecycler,
ownsIndexFn: ownsIndexFn,
}
info, err := opts.Resources.GetDocumentBuilders()
@@ -399,33 +401,6 @@ func (s *searchSupport) GetStats(ctx context.Context, req *resourcepb.ResourceSt
return rsp, nil
}
func (s *searchSupport) shouldBuildIndex(info ResourceStats) bool {
if s.ring == nil {
s.log.Debug("ring is not setup. Will proceed to build index")
return true
}
if s.ringLifecycler == nil {
s.log.Error("missing ring lifecycler")
return true
}
ringHasher := fnv.New32a()
_, err := ringHasher.Write([]byte(info.Namespace))
if err != nil {
s.log.Error("error hashing namespace", "namespace", info.Namespace, "err", err)
return true
}
rs, err := s.ring.GetWithOptions(ringHasher.Sum32(), searchOwnerRead, ring.WithReplicationFactor(s.ring.ReplicationFactor()))
if err != nil {
s.log.Error("error getting replicaset from ring", "namespace", info.Namespace, "err", err)
return true
}
return rs.Includes(s.ringLifecycler.GetInstanceAddr())
}
func (s *searchSupport) buildIndexes(ctx context.Context, rebuild bool) (int, error) {
totalBatchesIndexed := 0
group := errgroup.Group{}
@@ -442,7 +417,10 @@ func (s *searchSupport) buildIndexes(ctx context.Context, rebuild bool) (int, er
continue
}
if !s.shouldBuildIndex(info) {
own, err := s.ownsIndexFn(info.NamespacedResource)
if err != nil {
s.log.Warn("failed to check index ownership, building index", "namespace", info.Namespace, "group", info.Group, "resource", info.Resource, "error", err)
} else if !own {
s.log.Debug("skip building index", "namespace", info.Namespace, "group", info.Group, "resource", info.Resource)
continue
}
pkg/storage/unified/resource/search_server_distributor.go
-2
View File
@@ -90,8 +90,6 @@ var (
searchRingRead = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, func(s ring.InstanceState) bool {
return s != ring.ACTIVE
})
// operation used by the search-servers to check if they own the namespace
searchOwnerRead = ring.NewOp([]ring.InstanceState{ring.JOINING, ring.ACTIVE, ring.LEAVING}, nil)
)
func (ds *distributorServer) Search(ctx context.Context, r *resourcepb.ResourceSearchRequest) (*resourcepb.ResourceSearchResponse, error) {
pkg/storage/unified/resource/search_test.go
+3 -3
View File
@@ -213,7 +213,7 @@ func TestSearchGetOrCreateIndex(t *testing.T) {
InitMinCount: 1, // set min count to default for this test
}
support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil, nil, nil)
support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil, nil)
require.NoError(t, err)
require.NotNil(t, support)
@@ -274,7 +274,7 @@ func TestSearchGetOrCreateIndexWithIndexUpdate(t *testing.T) {
}
// Enable searchAfterWrite
support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil, nil, nil)
support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil, nil)
require.NoError(t, err)
require.NotNil(t, support)
@@ -325,7 +325,7 @@ func TestSearchGetOrCreateIndexWithCancellation(t *testing.T) {
InitMinCount: 1, // set min count to default for this test
}
support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil, nil, nil)
support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil, nil)
require.NoError(t, err)
require.NotNil(t, support)
pkg/storage/unified/resource/server.go
+2 -3
View File
@@ -235,8 +235,7 @@ type ResourceServerOptions struct {
QOSQueue QOSEnqueuer
QOSConfig QueueConfig
Ring *ring.Ring
RingLifecycler *ring.BasicLifecycler
OwnsIndexFn func(key NamespacedResource) (bool, error)
}
func NewResourceServer(opts ResourceServerOptions) (*server, error) {
@@ -334,7 +333,7 @@ func NewResourceServer(opts ResourceServerOptions) (*server, error) {
if opts.Search.Resources != nil {
var err error
s.search, err = newSearchSupport(opts.Search, s.backend, s.access, s.blob, opts.Tracer, opts.IndexMetrics, opts.Ring, opts.RingLifecycler)
s.search, err = newSearchSupport(opts.Search, s.backend, s.access, s.blob, opts.Tracer, opts.IndexMetrics, opts.OwnsIndexFn)
if err != nil {
return nil, err
}