public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve search index eviction (#111542)

Browse Source 
* Modify index eviction mechanism such that unowned indexes are also evicted.

* Propagate OwnsIndex function to bleve backend

Fix tests.

Stop eviction goroutine when stopping backend.

Make linter happy.

Make sure we stop backend created by tests.

Review suggestion.

Removed newline.
This commit is contained in:
Peter Štibraný
2025-09-24 16:54:35 +02:00
committed by GitHub
parent a4dabc31ed
commit 357aa7d314
13 changed files with 266 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/search/bleve.go
+106 -36
View File
@@ -28,6 +28,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.uber.org/atomic"
"k8s.io/apimachinery/pkg/selection"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
@@ -69,6 +70,7 @@ type BleveOptions struct {
BatchSize int
// Index cache TTL for bleve indices. 0 disables expiration for in-memory indexes.
// Also used for file-based indexes, if they are not owned by this instance, and they are not fetched from the cache recently.
IndexCacheTTL time.Duration
BuildVersion string
@@ -79,6 +81,11 @@ type BleveOptions struct {
Logger *slog.Logger
UseFullNgram bool
// This function is called to check whether the index is owned by the current instance.
// Indexes that are not owned by current instance are eligible for cleanup.
// If nil, all indexes are owned by the current instance.
OwnsIndex func(key resource.NamespacedResource) (bool, error)
}
type bleveBackend struct {
@@ -86,6 +93,9 @@ type bleveBackend struct {
log *slog.Logger
opts BleveOptions
// set from opts.OwnsIndex, always non-nil
ownsIndexFn func(key resource.NamespacedResource) (bool, error)
cacheMx sync.RWMutex
cache map[resource.NamespacedResource]*bleveIndex
@@ -94,8 +104,8 @@ type bleveBackend struct {
// if true will use ngram instead of edge_ngram for title indexes. See custom_analyzers.go
useFullNgram bool
metricsUpdaterCancel func()
metricsUpdaterWg sync.WaitGroup
bgTasksCancel func()
bgTasksWg sync.WaitGroup
}
func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, indexMetrics *resource.BleveIndexMetrics) (*bleveBackend, error) {
@@ -129,22 +139,31 @@ func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, indexMetrics *resou
log = slog.Default().With("logger", "bleve-backend")
}
ownFn := opts.OwnsIndex
if ownFn == nil {
// By default all indexes are owned by this instance.
ownFn = func(key resource.NamespacedResource) (bool, error) { return true, nil }
}
be := &bleveBackend{
log: log,
tracer: tracer,
cache: map[resource.NamespacedResource]*bleveIndex{},
opts: opts,
ownsIndexFn: ownFn,
indexMetrics: indexMetrics,
useFullNgram: opts.UseFullNgram,
}
ctx, cancel := context.WithCancel(context.Background())
be.bgTasksCancel = cancel
be.bgTasksWg.Add(1)
go be.evictExpiredOrUnownedIndexesPeriodically(ctx)
if be.indexMetrics != nil {
ctx, cancel := context.WithCancel(context.Background())
be.metricsUpdaterCancel = cancel
be.metricsUpdaterWg.Add(1)
be.bgTasksWg.Add(1)
go be.updateIndexSizeMetric(ctx, opts.Root)
} else {
be.metricsUpdaterCancel = func() { /* empty */ }
}
return be, nil
@@ -152,7 +171,7 @@ func NewBleveBackend(opts BleveOptions, tracer trace.Tracer, indexMetrics *resou
// GetIndex will return nil if the key does not exist
func (b *bleveBackend) GetIndex(_ context.Context, key resource.NamespacedResource) (resource.ResourceIndex, error) {
idx := b.getCachedIndex(key)
idx := b.getCachedIndex(key, time.Now())
// Avoid returning typed nils.
if idx == nil {
return nil, nil
@@ -160,48 +179,95 @@ func (b *bleveBackend) GetIndex(_ context.Context, key resource.NamespacedResour
return idx, nil
}
func (b *bleveBackend) getCachedIndex(key resource.NamespacedResource) *bleveIndex {
func (b *bleveBackend) getCachedIndex(key resource.NamespacedResource, now time.Time) *bleveIndex {
// Check index with read-lock first.
b.cacheMx.RLock()
val := b.cache[key]
idx := b.cache[key]
b.cacheMx.RUnlock()
if val == nil {
if idx == nil {
return nil
}
if val.expiration.IsZero() || val.expiration.After(time.Now()) {
// Not expired yet.
return val
}
idx.lastFetchedFromCache.Store(now.UnixMilli())
return idx
}
// We're dealing with expired index. We need to remove it from the cache and close it.
b.cacheMx.Lock()
val = b.cache[key]
delete(b.cache, key)
b.cacheMx.Unlock()
if val == nil {
return nil
}
// Index is no longer in the cache, but we need to close it.
err := val.stopUpdaterAndCloseIndex()
func (b *bleveBackend) closeIndex(idx *bleveIndex, key resource.NamespacedResource) {
err := idx.stopUpdaterAndCloseIndex()
if err != nil {
b.log.Error("failed to close index", "key", key, "err", err)
}
b.log.Info("index evicted from cache", "key", key)
if b.indexMetrics != nil {
b.indexMetrics.OpenIndexes.WithLabelValues(val.indexStorage).Dec()
b.indexMetrics.OpenIndexes.WithLabelValues(idx.indexStorage).Dec()
}
}
// This function will periodically evict expired or un-owned indexes from the cache.
func (b *bleveBackend) evictExpiredOrUnownedIndexesPeriodically(ctx context.Context) {
defer b.bgTasksWg.Done()
t := time.NewTicker(2 * time.Minute)
for ctx.Err() == nil {
select {
case <-ctx.Done():
return
case <-t.C:
b.runEvictExpiredOrUnownedIndexes(time.Now())
}
}
}
func (b *bleveBackend) runEvictExpiredOrUnownedIndexes(now time.Time) {
cacheTTLMillis := b.opts.IndexCacheTTL.Milliseconds()
// Collect all expired or unowned into this map, and perform the actual closing without holding the lock.
expired := map[resource.NamespacedResource]*bleveIndex{}
unowned := map[resource.NamespacedResource]*bleveIndex{}
ownCheckErrors := map[resource.NamespacedResource]error{}
b.cacheMx.Lock()
for key, idx := range b.cache {
// Check if index has expired.
if !idx.expiration.IsZero() && now.After(idx.expiration) {
delete(b.cache, key)
expired[key] = idx
continue
}
// Check if index is owned by this instance.
if cacheTTLMillis > 0 {
owned, err := b.ownsIndexFn(key)
if err != nil {
ownCheckErrors[key] = err
} else if !owned && now.UnixMilli()-idx.lastFetchedFromCache.Load() > cacheTTLMillis {
delete(b.cache, key)
unowned[key] = idx
}
}
}
b.cacheMx.Unlock()
for key, err := range ownCheckErrors {
b.log.Warn("failed to check if index belongs to this instance", "key", key, "err", err)
}
return nil
for key, idx := range unowned {
b.log.Info("index evicted from cache", "reason", "unowned", "key", key, "storage", idx.indexStorage)
b.closeIndex(idx, key)
}
for key, idx := range expired {
b.log.Info("index evicted from cache", "reason", "expired", "key", key, "storage", idx.indexStorage)
b.closeIndex(idx, key)
}
}
// updateIndexSizeMetric sets the total size of all file-based indices metric.
func (b *bleveBackend) updateIndexSizeMetric(ctx context.Context, indexPath string) {
defer b.metricsUpdaterWg.Done()
defer b.bgTasksWg.Done()
for ctx.Err() == nil {
var totalSize int64
@@ -357,7 +423,7 @@ func (b *bleveBackend) BuildIndex(
var index bleve.Index
var indexRV int64
cachedIndex := b.getCachedIndex(key)
cachedIndex := b.getCachedIndex(key, time.Now())
fileIndexName := "" // Name of the file-based index, or empty for in-memory indexes.
newIndexType := indexStorageMemory
build := true
@@ -572,9 +638,10 @@ func (b *bleveBackend) TotalDocs() int64 {
var totalDocs int64
// We iterate over keys and call getCachedIndex for each index individually.
// We do this to avoid keeping a lock for the entire TotalDocs function, since DocCount may be slow (due to disk access).
// Calling getCachedIndex also handles index expiration.
now := time.Now()
for _, key := range b.cacheKeys() {
idx := b.getCachedIndex(key)
idx := b.getCachedIndex(key, now)
if idx == nil {
continue
}
@@ -652,8 +719,8 @@ func (b *bleveBackend) findPreviousFileBasedIndex(resourceDir string, minBuildTi
func (b *bleveBackend) Stop() {
b.closeAllIndexes()
b.metricsUpdaterCancel()
b.metricsUpdaterWg.Wait()
b.bgTasksCancel()
b.bgTasksWg.Wait()
}
func (b *bleveBackend) closeAllIndexes() {
@@ -714,6 +781,9 @@ type bleveIndex struct {
updateLatency prometheus.Histogram
updatedDocuments prometheus.Summary
// Used to detect if the index can be safely closed, if it no longer belongs to this instance. UnixMilli.
lastFetchedFromCache atomic.Int64
}
func (b *bleveBackend) newBleveIndex(