[unified-storage/search] Don't expire file-based indexes, check for resource stats when building index on-demand (#107886)

* Get ResourceStats before indexing
* Replaced localcache.CacheService to handle expiration faster (localcache.CacheService / gocache.Cache only expires values at specific interval, but we need to close index faster)
* singleflight getOrBuildIndex for the same key
* expire only in-memory indexes
* file-based indexes have new name on each rebuild
* Sanitize file path segments, verify that generated path is within the root dir.
* Add comment and test for cleanOldIndexes.

pkg/storage/unified/resource/search_test.go

@@ -2,7 +2,9 @@ package resource
 
 import (
 	"context"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/grafana/authlib/types"
 	"github.com/stretchr/testify/mock"
@@ -96,6 +98,7 @@ func (m *mockStorageBackend) ListHistory(ctx context.Context, req *resourcepb.Li
 
 // mockSearchBackend implements SearchBackend for testing with tracking capabilities
 type mockSearchBackend struct {
+	mu                   sync.Mutex
 	buildIndexCalls      []buildIndexCall
 	buildEmptyIndexCalls []buildEmptyIndexCall
 }
@@ -129,6 +132,9 @@ func (m *mockSearchBackend) BuildIndex(ctx context.Context, key NamespacedResour
 		return nil, err
 	}
 
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
 	// Determine if this is an empty index based on size
 	// Empty indexes are characterized by size == 0
 	if size == 0 {
@@ -271,3 +277,58 @@ func TestBuildIndexes_MaxCountThreshold(t *testing.T) {
 		})
 	}
 }
+
+func TestSearchGetOrCreateIndex(t *testing.T) {
+	// Setup mock implementations
+	storage := &mockStorageBackend{
+		resourceStats: []ResourceStats{
+			{NamespacedResource: NamespacedResource{Namespace: "ns", Group: "group", Resource: "resource"}, Count: 50, ResourceVersion: 11111111},
+		},
+	}
+	search := &mockSearchBackend{
+		buildIndexCalls:      []buildIndexCall{},
+		buildEmptyIndexCalls: []buildEmptyIndexCall{},
+	}
+	supplier := &TestDocumentBuilderSupplier{
+		GroupsResources: map[string]string{
+			"group": "resource",
+		},
+	}
+
+	// Create search support with the specified initMaxSize
+	opts := SearchOptions{
+		Backend:       search,
+		Resources:     supplier,
+		WorkerThreads: 1,
+		InitMinCount:  1, // set min count to default for this test
+		InitMaxCount:  0,
+	}
+
+	support, err := newSearchSupport(opts, storage, nil, nil, noop.NewTracerProvider().Tracer("test"), nil)
+	require.NoError(t, err)
+	require.NotNil(t, support)
+
+	start := make(chan struct{})
+
+	const concurrency = 100
+	wg := sync.WaitGroup{}
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			<-start
+			_, _ = support.getOrCreateIndex(context.Background(), NamespacedResource{Namespace: "ns", Group: "group", Resource: "resource"})
+		}()
+	}
+
+	// Wait a bit for goroutines to start (hopefully)
+	time.Sleep(10 * time.Millisecond)
+	// Unblock all goroutines.
+	close(start)
+	wg.Wait()
+
+	require.NotEmpty(t, search.buildIndexCalls)
+	require.Less(t, len(search.buildIndexCalls), concurrency, "Should not have built index more than a few times (ideally once)")
+	require.Equal(t, int64(50), search.buildIndexCalls[0].size)
+	require.Equal(t, int64(11111111), search.buildIndexCalls[0].resourceVersion)
+}