From f77e99d9652e14e410772ed6b56777e5cf2c6f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Tue, 16 Sep 2025 10:52:30 +0200 Subject: [PATCH] Store build time and build version into index. (#111010) --- pkg/storage/unified/search/bleve.go | 58 ++++++++++++++++++++---- pkg/storage/unified/search/bleve_test.go | 21 +++++++++ pkg/storage/unified/search/options.go | 2 + 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index d9c7ac95fc8..58b422176b4 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -47,6 +47,12 @@ const ( indexStorageFile = "file" ) +// Keys used to store internal data in index. +const ( + internalRVKey = "rv" // Encoded as big-endian int64 + internalBuildInfoKey = "build_info" // Encoded as JSON of IndexBuildInfo struct +) + var _ resource.SearchBackend = &bleveBackend{} var _ resource.ResourceIndex = &bleveIndex{} @@ -64,6 +70,8 @@ type BleveOptions struct { // Index cache TTL for bleve indices. 0 disables expiration for in-memory indexes. IndexCacheTTL time.Duration + BuildVersion string + Logger *slog.Logger } @@ -199,13 +207,38 @@ func (b *bleveBackend) updateIndexSizeMetric(indexPath string) { // newBleveIndex creates a new bleve index with consistent configuration. // If path is empty, creates an in-memory index. // If path is not empty, creates a file-based index at the specified path. -func newBleveIndex(path string, mapper mapping.IndexMapping) (bleve.Index, error) { +func newBleveIndex(path string, mapper mapping.IndexMapping, buildTime time.Time, buildVersion string) (bleve.Index, error) { kvstore := bleve.Config.DefaultKVStore if path == "" { // use in-memory kvstore kvstore = bleve.Config.DefaultMemKVStore } - return bleve.NewUsing(path, mapper, bleve.Config.DefaultIndexType, kvstore, nil) + ix, err := bleve.NewUsing(path, mapper, bleve.Config.DefaultIndexType, kvstore, nil) + if err != nil { + return nil, err + } + + bi := IndexBuildInfo{ + BuildTime: buildTime.Unix(), + BuildVersion: buildVersion, + } + + biBytes, err := json.Marshal(bi) + if err != nil { + cErr := ix.Close() + return nil, errors.Join(fmt.Errorf("failed to store index build info: %w", err), cErr) + } + + if err = ix.SetInternal([]byte(internalBuildInfoKey), biBytes); err != nil { + cErr := ix.Close() + return nil, errors.Join(fmt.Errorf("failed to store index build info: %w", err), cErr) + } + return ix, nil +} + +type IndexBuildInfo struct { + BuildTime int64 `json:"build_time"` // Unix seconds timestamp of time when the index was built + BuildVersion string `json:"build_version"` // Grafana version used when building the index } // BuildIndex builds an index from scratch or retrieves it from the filesystem. @@ -304,7 +337,7 @@ func (b *bleveBackend) BuildIndex( return nil, fmt.Errorf("invalid path %s", indexDir) } - index, err = newBleveIndex(indexDir, mapper) + index, err = newBleveIndex(indexDir, mapper, time.Now(), b.opts.BuildVersion) if errors.Is(err, bleve.ErrorIndexPathExists) { now = now.Add(time.Second) // Bump time for next try index = nil // Bleve actually returns non-nil value with ErrorIndexPathExists @@ -319,7 +352,7 @@ func (b *bleveBackend) BuildIndex( defer closeIndexOnExit(index, indexDir) // Close index, and delete new index directory. } } else { - index, err = newBleveIndex("", mapper) + index, err = newBleveIndex("", mapper, time.Now(), b.opts.BuildVersion) if err != nil { return nil, fmt.Errorf("error creating new in-memory bleve index: %w", err) } @@ -652,8 +685,6 @@ func (b *bleveIndex) BulkIndex(req *resource.BulkIndexRequest) error { return b.index.Batch(batch) } -var internalRVKey = []byte("rv") - func (b *bleveIndex) updateResourceVersion(rv int64) error { if rv == 0 { return nil @@ -672,13 +703,13 @@ func setRV(index bleve.Index, rv int64) error { buf := make([]byte, 8) binary.BigEndian.PutUint64(buf, uint64(rv)) - return index.SetInternal(internalRVKey, buf) + return index.SetInternal([]byte(internalRVKey), buf) } // getRV will call index.GetInternal to retrieve the RV saved in the index. If index is closed, it will return a // bleve.ErrorIndexClosed error. If there's no RV saved in the index, or it's invalid format, it will return 0 func getRV(index bleve.Index) (int64, error) { - raw, err := index.GetInternal(internalRVKey) + raw, err := index.GetInternal([]byte(internalRVKey)) if err != nil { return 0, err } @@ -690,6 +721,17 @@ func getRV(index bleve.Index) (int64, error) { return int64(binary.BigEndian.Uint64(raw)), nil } +func getBuildInfo(index bleve.Index) (IndexBuildInfo, error) { + raw, err := index.GetInternal([]byte(internalBuildInfoKey)) + if err != nil { + return IndexBuildInfo{}, err + } + + res := IndexBuildInfo{} + err = json.Unmarshal(raw, &res) + return res, err +} + func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.ListManagedObjectsRequest) (*resourcepb.ListManagedObjectsResponse, error) { if req.NextPageToken != "" { return nil, fmt.Errorf("next page not implemented yet") diff --git a/pkg/storage/unified/search/bleve_test.go b/pkg/storage/unified/search/bleve_test.go index f270c375e86..2aecb6a20d8 100644 --- a/pkg/storage/unified/search/bleve_test.go +++ b/pkg/storage/unified/search/bleve_test.go @@ -756,6 +756,8 @@ func Test_isPathWithinRoot(t *testing.T) { } } +const buildVersion = "1.2.3-456" + func setupBleveBackend(t *testing.T, fileThreshold int, cacheTTL time.Duration, dir string) (*bleveBackend, prometheus.Gatherer) { if dir == "" { dir = t.TempDir() @@ -768,6 +770,7 @@ func setupBleveBackend(t *testing.T, fileThreshold int, cacheTTL time.Duration, FileThreshold: int64(fileThreshold), IndexCacheTTL: cacheTTL, Logger: slog.New(logtest.NewNopHandler(t)), + BuildVersion: buildVersion, }, tracing.NewNoopTracerService(), metrics) require.NoError(t, err) require.NotNil(t, backend) @@ -1398,6 +1401,24 @@ func TestIndexUpdateWithErrors(t *testing.T) { }) } +func TestIndexBuildInfo(t *testing.T) { + ns := resource.NamespacedResource{ + Namespace: "test", + Group: "group", + Resource: "resource", + } + + be, _ := setupBleveBackend(t, 100, 1*time.Minute, "") + index, err := be.BuildIndex(t.Context(), ns, 10, nil, "test", indexTestDocs(ns, 10, 100), nil, false) + require.NoError(t, err) + + buildInfo, err := getBuildInfo(index.(*bleveIndex).index) + require.NoError(t, err) + require.NotNil(t, buildInfo) + require.Equal(t, buildVersion, buildInfo.BuildVersion) + require.InDelta(t, float64(time.Now().Unix()), buildInfo.BuildTime, 30) // allow 30 seconds of drift +} + func searchTitle(t *testing.T, idx resource.ResourceIndex, query string, limit int, ns resource.NamespacedResource) *resourcepb.ResourceSearchResponse { resp, err := idx.Search(t.Context(), nil, &resourcepb.ResourceSearchRequest{ Options: &resourcepb.ListOptions{ diff --git a/pkg/storage/unified/search/options.go b/pkg/storage/unified/search/options.go index b50f68ddf4d..893debf6bfc 100644 --- a/pkg/storage/unified/search/options.go +++ b/pkg/storage/unified/search/options.go @@ -23,11 +23,13 @@ func NewSearchOptions(features featuremgmt.FeatureToggles, cfg *setting.Cfg, tra if err != nil { return resource.SearchOptions{}, err } + bleve, err := NewBleveBackend(BleveOptions{ Root: root, FileThreshold: int64(cfg.IndexFileThreshold), // fewer than X items will use a memory index BatchSize: cfg.IndexMaxBatchSize, // This is the batch size for how many objects to add to the index at once IndexCacheTTL: cfg.IndexCacheTTL, // How long to keep the index cache in memory + BuildVersion: cfg.BuildVersion, }, tracer, indexMetrics) if err != nil {