Search: Build index from resource stats (#97320)

pkg/storage/unified/resource/cdk_backend.go

@@ -108,7 +108,8 @@ func (s *cdkBackend) getPath(key *ResourceKey, rv int64) string {
 	return buffer.String()
 }
 
-func (s *cdkBackend) Namespaces(ctx context.Context) ([]string, error) {
+// GetResourceStats implements Backend.
+func (s *cdkBackend) GetResourceStats(ctx context.Context, minCount int) ([]ResourceStats, error) {
 	return nil, fmt.Errorf("not implemented")
 }
 

pkg/storage/unified/resource/search.go

@@ -81,6 +81,7 @@ type searchSupport struct {
 	access      authz.AccessClient
 	builders    *builderCache
 	initWorkers int
+	initMinSize int
 }
 
 var (
@@ -104,6 +105,7 @@ func newSearchSupport(opts SearchOptions, storage StorageBackend, access authz.A
 		search:      opts.Backend,
 		log:         slog.Default().With("logger", "resource-search"),
 		initWorkers: opts.WorkerThreads,
+		initMinSize: opts.InitMinCount,
 	}
 
 	info, err := opts.Resources.GetDocumentBuilders()
@@ -164,40 +166,22 @@ func (s *searchSupport) init(ctx context.Context) error {
 	_, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Init")
 	defer span.End()
 
-	// TODO, replace namespaces with a query that gets top values
-	namespaces, err := s.storage.Namespaces(ctx)
-	if err != nil {
-		return err
-	}
-
-	// Hardcoded for now... should come from the query
-	kinds := []schema.GroupResource{
-		{Group: "dashboard.grafana.app", Resource: "dashboards"},
-		{Group: "playlist.grafana.app", Resource: "playlists"},
-	}
-
 	totalBatchesIndexed := 0
 	group := errgroup.Group{}
 	group.SetLimit(s.initWorkers)
 
-	// Prepare all the (large) indexes
-	// TODO, threading and query real information:
-	// SELECT namespace,"group",resource,COUNT(*),resource_version FROM resource
-	//   GROUP BY "group", "resource", "namespace"
-	//   ORDER BY resource_version desc;
-	for _, ns := range namespaces {
-		for _, gr := range kinds {
-			group.Go(func() error {
-				s.log.Debug("initializing search index", "namespace", ns, "gr", gr)
-				totalBatchesIndexed++
-				_, _, err = s.build(ctx, NamespacedResource{
-					Group:     gr.Group,
-					Resource:  gr.Resource,
-					Namespace: ns,
-				}, 10, 0) // TODO, approximate size
-				return err
-			})
-		}
+	stats, err := s.storage.GetResourceStats(ctx, s.initMinSize)
+	if err != nil {
+		return err
+	}
+
+	for _, info := range stats {
+		group.Go(func() error {
+			s.log.Debug("initializing search index", "namespace", info.Namespace, "group", info.Group, "resource", info.Resource)
+			totalBatchesIndexed++
+			_, _, err = s.build(ctx, info.NamespacedResource, info.Count, info.ResourceVersion)
+			return err
+		})
 	}
 
 	err = group.Wait()

pkg/storage/unified/resource/server.go

@@ -95,7 +95,14 @@ type StorageBackend interface {
 	// For HA setups, this will be more events than the local WriteEvent above!
 	WatchWriteEvents(ctx context.Context) (<-chan *WrittenEvent, error)
 
-	Namespaces(ctx context.Context) ([]string, error)
+	GetResourceStats(ctx context.Context, minCount int) ([]ResourceStats, error)
+}
+
+type ResourceStats struct {
+	NamespacedResource
+
+	Count           int64
+	ResourceVersion int64
 }
 
 // This interface is not exposed to end users directly
@@ -133,6 +140,9 @@ type SearchOptions struct {
 
 	// How many threads should build indexes
 	WorkerThreads int
+
+	// Skip building index on startup for small indexes
+	InitMinCount int
 }
 
 type ResourceServerOptions struct {