Search: Return counts for values within an folder/repository (#97534)

pkg/apis/folder/v0alpha1/types.go

@@ -68,5 +68,11 @@ type FolderAccessInfo struct {
 type DescendantCounts struct {
 	metav1.TypeMeta `json:",inline"`
 
-	Counts map[string]int64 `json:"counts"`
+	Counts []ResourceStats `json:"counts"`
+}
+
+type ResourceStats struct {
+	Group    string `json:"group"`
+	Resource string `json:"resource"`
+	Count    int64  `json:"count"`
 }

pkg/apis/folder/v0alpha1/zz_generated.deepcopy.go

@@ -17,10 +17,8 @@ func (in *DescendantCounts) DeepCopyInto(out *DescendantCounts) {
 	out.TypeMeta = in.TypeMeta
 	if in.Counts != nil {
 		in, out := &in.Counts, &out.Counts
-		*out = make(map[string]int64, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
+		*out = make([]ResourceStats, len(*in))
+		copy(*out, *in)
 	}
 	return
 }
@@ -175,6 +173,22 @@ func (in *FolderList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResourceStats) DeepCopyInto(out *ResourceStats) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceStats.
+func (in *ResourceStats) DeepCopy() *ResourceStats {
+	if in == nil {
+		return nil
+	}
+	out := new(ResourceStats)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Spec) DeepCopyInto(out *Spec) {
 	*out = *in

pkg/apis/folder/v0alpha1/zz_generated.openapi.go

@@ -20,6 +20,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 		"github.com/grafana/grafana/pkg/apis/folder/v0alpha1.FolderInfo":       schema_pkg_apis_folder_v0alpha1_FolderInfo(ref),
 		"github.com/grafana/grafana/pkg/apis/folder/v0alpha1.FolderInfoList":   schema_pkg_apis_folder_v0alpha1_FolderInfoList(ref),
 		"github.com/grafana/grafana/pkg/apis/folder/v0alpha1.FolderList":       schema_pkg_apis_folder_v0alpha1_FolderList(ref),
+		"github.com/grafana/grafana/pkg/apis/folder/v0alpha1.ResourceStats":    schema_pkg_apis_folder_v0alpha1_ResourceStats(ref),
 		"github.com/grafana/grafana/pkg/apis/folder/v0alpha1.Spec":             schema_pkg_apis_folder_v0alpha1_Spec(ref),
 	}
 }
@@ -46,14 +47,12 @@ func schema_pkg_apis_folder_v0alpha1_DescendantCounts(ref common.ReferenceCallba
 					},
 					"counts": {
 						SchemaProps: spec.SchemaProps{
-							Type: []string{"object"},
-							AdditionalProperties: &spec.SchemaOrBool{
-								Allows: true,
+							Type: []string{"array"},
+							Items: &spec.SchemaOrArray{
 								Schema: &spec.Schema{
 									SchemaProps: spec.SchemaProps{
-										Default: 0,
-										Type:    []string{"integer"},
-										Format:  "int64",
+										Default: map[string]interface{}{},
+										Ref:     ref("github.com/grafana/grafana/pkg/apis/folder/v0alpha1.ResourceStats"),
 									},
 								},
 							},
@@ -63,6 +62,8 @@ func schema_pkg_apis_folder_v0alpha1_DescendantCounts(ref common.ReferenceCallba
 				Required: []string{"counts"},
 			},
 		},
+		Dependencies: []string{
+			"github.com/grafana/grafana/pkg/apis/folder/v0alpha1.ResourceStats"},
 	}
 }
 
@@ -302,6 +303,40 @@ func schema_pkg_apis_folder_v0alpha1_FolderList(ref common.ReferenceCallback) co
 	}
 }
 
+func schema_pkg_apis_folder_v0alpha1_ResourceStats(ref common.ReferenceCallback) common.OpenAPIDefinition {
+	return common.OpenAPIDefinition{
+		Schema: spec.Schema{
+			SchemaProps: spec.SchemaProps{
+				Type: []string{"object"},
+				Properties: map[string]spec.Schema{
+					"group": {
+						SchemaProps: spec.SchemaProps{
+							Default: "",
+							Type:    []string{"string"},
+							Format:  "",
+						},
+					},
+					"resource": {
+						SchemaProps: spec.SchemaProps{
+							Default: "",
+							Type:    []string{"string"},
+							Format:  "",
+						},
+					},
+					"count": {
+						SchemaProps: spec.SchemaProps{
+							Default: 0,
+							Type:    []string{"integer"},
+							Format:  "int64",
+						},
+					},
+				},
+				Required: []string{"group", "resource", "count"},
+			},
+		},
+	}
+}
+
 func schema_pkg_apis_folder_v0alpha1_Spec(ref common.ReferenceCallback) common.OpenAPIDefinition {
 	return common.OpenAPIDefinition{
 		Schema: spec.Schema{

pkg/apis/folder/v0alpha1/zz_generated.openapi_violation_exceptions.list

@@ -1 +1,2 @@
+API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/folder/v0alpha1,DescendantCounts,Counts
 API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/folder/v0alpha1,FolderInfoList,Items

pkg/registry/apis/dashboard/legacy/sql_dashboards.go

@@ -14,7 +14,6 @@ import (
 	"k8s.io/utils/ptr"
 
 	"github.com/grafana/authlib/claims"
-
 	"github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/apimachinery/utils"

pkg/registry/apis/dashboard/legacy/storage.go

@@ -333,3 +333,8 @@ func (a *dashboardSqlAccess) History(ctx context.Context, req *resource.HistoryR
 func (a *dashboardSqlAccess) Origin(context.Context, *resource.OriginRequest) (*resource.OriginResponse, error) {
 	return nil, fmt.Errorf("not yet (origin)")
 }
+
+// GetStats implements ResourceServer.
+func (a *dashboardSqlAccess) GetStats(ctx context.Context, req *resource.ResourceStatsRequest) (*resource.ResourceStatsResponse, error) {
+	return nil, fmt.Errorf("not yet (GetStats)")
+}

pkg/registry/apis/folders/register.go

@@ -17,12 +17,11 @@ import (
 	common "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
 
-	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
-
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	"github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
 	grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
+	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/apiserver/builder"
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
@@ -30,6 +29,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/storage/unified/resource"
 )
 
 var _ builder.APIGroupBuilder = (*FolderAPIBuilder)(nil)
@@ -48,6 +48,7 @@ type FolderAPIBuilder struct {
 	folderSvc     folder.Service
 	storage       grafanarest.Storage
 	accessControl accesscontrol.AccessControl
+	searcher      resource.ResourceIndexClient
 }
 
 func RegisterAPIService(cfg *setting.Cfg,
@@ -56,10 +57,12 @@ func RegisterAPIService(cfg *setting.Cfg,
 	folderSvc folder.Service,
 	accessControl accesscontrol.AccessControl,
 	registerer prometheus.Registerer,
+	unified resource.ResourceClient,
 ) *FolderAPIBuilder {
 	if !featuremgmt.AnyEnabled(features,
 		featuremgmt.FlagKubernetesFolders,
 		featuremgmt.FlagGrafanaAPIServerTestingWithExperimentalAPIs,
+		featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs,
 		featuremgmt.FlagProvisioning) {
 		return nil // skip registration unless opting into Kubernetes folders or unless we want to customize registration when testing
 	}
@@ -70,6 +73,7 @@ func RegisterAPIService(cfg *setting.Cfg,
 		namespacer:    request.GetNamespaceMapper(cfg),
 		folderSvc:     folderSvc,
 		accessControl: accessControl,
+		searcher:      unified,
 	}
 	apiregistration.RegisterAPI(builder)
 	return builder
@@ -122,8 +126,10 @@ func (b *FolderAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.API
 	storage := map[string]rest.Storage{}
 	storage[resourceInfo.StoragePath()] = legacyStore
 	storage[resourceInfo.StoragePath("parents")] = &subParentsREST{b.folderSvc}
-	storage[resourceInfo.StoragePath("count")] = &subCountREST{b.folderSvc}
 	storage[resourceInfo.StoragePath("access")] = &subAccessREST{b.folderSvc}
+	storage[resourceInfo.StoragePath("count")] = &subCountREST{
+		searcher: b.searcher,
+	}
 
 	// enable dual writer
 	if optsGetter != nil && dualWriteBuilder != nil {
@@ -242,6 +248,9 @@ func (b *FolderAPIBuilder) Validate(ctx context.Context, a admission.Attributes,
 	}
 
 	obj := a.GetObject()
+	if obj == nil || a.GetOperation() == admission.Connect {
+		return nil // This is normal for sub-resource
+	}
 
 	f, ok := obj.(*v0alpha1.Folder)
 	if !ok {

pkg/registry/apis/folders/sub_count.go

@@ -7,14 +7,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/rest"
 
-	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
-	"github.com/grafana/grafana/pkg/services/folder"
+	"github.com/grafana/grafana/pkg/storage/unified/resource"
 )
 
 type subCountREST struct {
-	service folder.Service
+	searcher resource.ResourceIndexClient
 }
 
 var (
@@ -46,11 +45,6 @@ func (r *subCountREST) NewConnectOptions() (runtime.Object, bool, string) {
 }
 
 func (r *subCountREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
-	user, err := identity.GetRequester(ctx)
-	if err != nil {
-		return nil, err
-	}
-
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		ns, err := request.NamespaceInfoFrom(ctx, true)
 		if err != nil {
@@ -58,18 +52,24 @@ func (r *subCountREST) Connect(ctx context.Context, name string, opts runtime.Ob
 			return
 		}
 
-		counts, err := r.service.GetDescendantCounts(ctx, &folder.GetDescendantCountsQuery{
-			UID:          &name,
-			OrgID:        ns.OrgID,
-			SignedInUser: user,
+		stats, err := r.searcher.GetStats(ctx, &resource.ResourceStatsRequest{
+			Namespace: ns.Value,
+			Folder:    name,
 		})
 		if err != nil {
 			responder.Error(err)
 			return
 		}
-
-		responder.Object(http.StatusOK, &v0alpha1.DescendantCounts{
-			Counts: counts,
-		})
+		rsp := &v0alpha1.DescendantCounts{
+			Counts: make([]v0alpha1.ResourceStats, len(stats.Stats)),
+		}
+		for i, v := range stats.Stats {
+			rsp.Counts[i] = v0alpha1.ResourceStats{
+				Group:    v.Group,
+				Resource: v.Resource,
+				Count:    v.Count,
+			}
+		}
+		responder.Object(200, rsp)
 	}), nil
 }

pkg/storage/unified/resource/resource.proto

@@ -323,6 +323,38 @@ message WatchEvent {
   Resource previous = 4;
 }
 
+// Get statistics across multiple resources
+// For these queries, we do not need authorization to see the actual values
+message ResourceStatsRequest {
+  // Namespace (tenant)
+  string namespace = 1;
+
+  // An optional list of group/resource identifiers
+  // when empty, we assume searching across everything
+  // NOTE, this query may need to federate across a few storage instances
+  repeated string kinds = 2;
+
+  // Limit the stats within a folder (not recursive!) 
+  string folder = 3;
+}
+
+message ResourceStatsResponse {
+  message Stats {
+    // Resource group
+    string group = 1;
+    // Resource name
+    string resource = 2;
+    // Number of items
+    int64 count = 3;
+  }
+
+  // Error details
+  ErrorResult error = 1;
+
+  // All results exist within this key
+  repeated Stats stats = 2;
+}
+
 // Search within a single resource
 message ResourceSearchRequest {
   message Sort {
@@ -715,6 +747,9 @@ service ResourceStore {
 service ResourceIndex {
   rpc Search(ResourceSearchRequest) returns (ResourceSearchResponse);
 
+  // Get the resource stats
+  rpc GetStats(ResourceStatsRequest) returns (ResourceStatsResponse);
+
   // Show resource history (and trash)
   rpc History(HistoryRequest) returns (HistoryResponse);
 

pkg/storage/unified/resource/resource_grpc.pb.go

@@ -348,9 +348,10 @@ var ResourceStore_ServiceDesc = grpc.ServiceDesc{
 }
 
 const (
-	ResourceIndex_Search_FullMethodName  = "/resource.ResourceIndex/Search"
-	ResourceIndex_History_FullMethodName = "/resource.ResourceIndex/History"
-	ResourceIndex_Origin_FullMethodName  = "/resource.ResourceIndex/Origin"
+	ResourceIndex_Search_FullMethodName   = "/resource.ResourceIndex/Search"
+	ResourceIndex_GetStats_FullMethodName = "/resource.ResourceIndex/GetStats"
+	ResourceIndex_History_FullMethodName  = "/resource.ResourceIndex/History"
+	ResourceIndex_Origin_FullMethodName   = "/resource.ResourceIndex/Origin"
 )
 
 // ResourceIndexClient is the client API for ResourceIndex service.
@@ -361,6 +362,8 @@ const (
 // It should be implemented with efficient indexes and does not need read-after-write semantics
 type ResourceIndexClient interface {
 	Search(ctx context.Context, in *ResourceSearchRequest, opts ...grpc.CallOption) (*ResourceSearchResponse, error)
+	// Get the resource stats
+	GetStats(ctx context.Context, in *ResourceStatsRequest, opts ...grpc.CallOption) (*ResourceStatsResponse, error)
 	// Show resource history (and trash)
 	History(ctx context.Context, in *HistoryRequest, opts ...grpc.CallOption) (*HistoryResponse, error)
 	// Used for efficient provisioning
@@ -385,6 +388,16 @@ func (c *resourceIndexClient) Search(ctx context.Context, in *ResourceSearchRequ
 	return out, nil
 }
 
+func (c *resourceIndexClient) GetStats(ctx context.Context, in *ResourceStatsRequest, opts ...grpc.CallOption) (*ResourceStatsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ResourceStatsResponse)
+	err := c.cc.Invoke(ctx, ResourceIndex_GetStats_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *resourceIndexClient) History(ctx context.Context, in *HistoryRequest, opts ...grpc.CallOption) (*HistoryResponse, error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(HistoryResponse)
@@ -413,6 +426,8 @@ func (c *resourceIndexClient) Origin(ctx context.Context, in *OriginRequest, opt
 // It should be implemented with efficient indexes and does not need read-after-write semantics
 type ResourceIndexServer interface {
 	Search(context.Context, *ResourceSearchRequest) (*ResourceSearchResponse, error)
+	// Get the resource stats
+	GetStats(context.Context, *ResourceStatsRequest) (*ResourceStatsResponse, error)
 	// Show resource history (and trash)
 	History(context.Context, *HistoryRequest) (*HistoryResponse, error)
 	// Used for efficient provisioning
@@ -426,6 +441,9 @@ type UnimplementedResourceIndexServer struct {
 func (UnimplementedResourceIndexServer) Search(context.Context, *ResourceSearchRequest) (*ResourceSearchResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
 }
+func (UnimplementedResourceIndexServer) GetStats(context.Context, *ResourceStatsRequest) (*ResourceStatsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetStats not implemented")
+}
 func (UnimplementedResourceIndexServer) History(context.Context, *HistoryRequest) (*HistoryResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method History not implemented")
 }
@@ -462,6 +480,24 @@ func _ResourceIndex_Search_Handler(srv interface{}, ctx context.Context, dec fun
 	return interceptor(ctx, in, info, handler)
 }
 
+func _ResourceIndex_GetStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ResourceStatsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ResourceIndexServer).GetStats(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: ResourceIndex_GetStats_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ResourceIndexServer).GetStats(ctx, req.(*ResourceStatsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _ResourceIndex_History_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(HistoryRequest)
 	if err := dec(in); err != nil {
@@ -509,6 +545,10 @@ var ResourceIndex_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "Search",
 			Handler:    _ResourceIndex_Search_Handler,
 		},
+		{
+			MethodName: "GetStats",
+			Handler:    _ResourceIndex_GetStats_Handler,
+		},
 		{
 			MethodName: "History",
 			Handler:    _ResourceIndex_History_Handler,

pkg/storage/unified/resource/search.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log/slog"
 	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -47,7 +48,7 @@ type ResourceIndex interface {
 	Origin(ctx context.Context, req *OriginRequest) (*OriginResponse, error)
 
 	// Get the number of documents in the index
-	DocCount() (int, error)
+	DocCount(ctx context.Context, folder string) (int64, error)
 }
 
 // SearchBackend contains the technology specific logic to support search
@@ -168,6 +169,87 @@ func (s *searchSupport) Search(ctx context.Context, req *ResourceSearchRequest)
 	return idx.Search(ctx, s.access, req, federate)
 }
 
+// GetStats implements ResourceServer.
+func (s *searchSupport) GetStats(ctx context.Context, req *ResourceStatsRequest) (*ResourceStatsResponse, error) {
+	if req.Namespace == "" {
+		return &ResourceStatsResponse{
+			Error: NewBadRequestError("missing namespace"),
+		}, nil
+	}
+	rsp := &ResourceStatsResponse{}
+
+	// Explicit list of kinds
+	if len(req.Kinds) > 0 {
+		rsp.Stats = make([]*ResourceStatsResponse_Stats, len(req.Kinds))
+		for i, k := range req.Kinds {
+			parts := strings.SplitN(k, "/", 2)
+			index, err := s.getOrCreateIndex(ctx, NamespacedResource{
+				Namespace: req.Namespace,
+				Group:     parts[0],
+				Resource:  parts[1],
+			})
+			if err != nil {
+				rsp.Error = AsErrorResult(err)
+				return rsp, nil
+			}
+			count, err := index.DocCount(ctx, req.Folder)
+			if err != nil {
+				rsp.Error = AsErrorResult(err)
+				return rsp, nil
+			}
+			rsp.Stats[i] = &ResourceStatsResponse_Stats{
+				Group:    parts[0],
+				Resource: parts[1],
+				Count:    count,
+			}
+		}
+		return rsp, nil
+	}
+
+	stats, err := s.storage.GetResourceStats(ctx, req.Namespace, 0)
+	if err != nil {
+		return &ResourceStatsResponse{
+			Error: AsErrorResult(err),
+		}, nil
+	}
+	rsp.Stats = make([]*ResourceStatsResponse_Stats, len(stats))
+
+	// When not filtered by folder or repository, we can use the results directly
+	if req.Folder == "" {
+		for i, stat := range stats {
+			rsp.Stats[i] = &ResourceStatsResponse_Stats{
+				Group:    stat.Group,
+				Resource: stat.Resource,
+				Count:    stat.Count,
+			}
+		}
+		return rsp, nil
+	}
+
+	for i, stat := range stats {
+		index, err := s.getOrCreateIndex(ctx, NamespacedResource{
+			Namespace: req.Namespace,
+			Group:     stat.Group,
+			Resource:  stat.Resource,
+		})
+		if err != nil {
+			rsp.Error = AsErrorResult(err)
+			return rsp, nil
+		}
+		count, err := index.DocCount(ctx, req.Folder)
+		if err != nil {
+			rsp.Error = AsErrorResult(err)
+			return rsp, nil
+		}
+		rsp.Stats[i] = &ResourceStatsResponse_Stats{
+			Group:    stat.Group,
+			Resource: stat.Resource,
+			Count:    count,
+		}
+	}
+	return rsp, nil
+}
+
 // init is called during startup.  any failure will block startup and continued execution
 func (s *searchSupport) init(ctx context.Context) error {
 	ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Init")
@@ -360,7 +442,7 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
 	}
 
 	// Record the number of objects indexed for the kind/resource
-	docCount, err := index.DocCount()
+	docCount, err := index.DocCount(ctx, "")
 	if err != nil {
 		s.log.Warn("error getting doc count", "error", err)
 	}

pkg/storage/unified/resource/server.go

@@ -920,6 +920,22 @@ func (s *server) Search(ctx context.Context, req *ResourceSearchRequest) (*Resou
 	return s.search.Search(ctx, req)
 }
 
+// GetStats implements ResourceServer.
+func (s *server) GetStats(ctx context.Context, req *ResourceStatsRequest) (*ResourceStatsResponse, error) {
+	if err := s.Init(ctx); err != nil {
+		return nil, err
+	}
+	if s.search == nil {
+		// If the backend implements "GetStats", we can use it
+		srv, ok := s.backend.(ResourceIndexServer)
+		if ok {
+			return srv.GetStats(ctx, req)
+		}
+		return nil, fmt.Errorf("search index not configured")
+	}
+	return s.search.GetStats(ctx, req)
+}
+
 // History implements ResourceServer.
 func (s *server) History(ctx context.Context, req *HistoryRequest) (*HistoryResponse, error) {
 	return s.search.History(ctx, req)

pkg/storage/unified/search/bleve.go

@@ -251,7 +251,7 @@ func (b *bleveIndex) Search(
 		searchrequest.Fields = f
 	}
 
-	res, err := index.Search(searchrequest)
+	res, err := index.SearchInContext(ctx, searchrequest)
 	if err != nil {
 		return nil, err
 	}
@@ -294,9 +294,25 @@ func (b *bleveIndex) Search(
 	return response, nil
 }
 
-func (b *bleveIndex) DocCount() (int, error) {
-	count, err := b.index.DocCount()
-	return int(count), err
+func (b *bleveIndex) DocCount(ctx context.Context, folder string) (int64, error) {
+	if folder == "" {
+		count, err := b.index.DocCount()
+		return int64(count), err
+	}
+
+	req := &bleve.SearchRequest{
+		Size:   0, // we just need the count
+		Fields: []string{},
+		Query: &query.TermQuery{
+			Term:     folder,
+			FieldVal: resource.SEARCH_FIELD_FOLDER,
+		},
+	}
+	rsp, err := b.index.SearchInContext(ctx, req)
+	if rsp == nil {
+		return 0, err
+	}
+	return int64(rsp.Total), err
 }
 
 // make sure the request key matches the index

pkg/storage/unified/search/bleve_mappings.go

@@ -17,7 +17,7 @@ func getBleveMappings(fields resource.SearchableDocumentFields) mapping.IndexMap
 func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentMapping {
 	mapper := bleve.NewDocumentStaticMapping()
 	mapper.AddFieldMapping(&mapping.FieldMapping{
-		Name: "title",
+		Name: resource.SEARCH_FIELD_TITLE,
 		Type: "text",
 		// TODO - if we don't want title to be a keyword, we can use this
 		// set the title field to use keyword analyzer so it sorts by the whole phrase
@@ -31,7 +31,7 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
 	})
 
 	mapper.AddFieldMapping(&mapping.FieldMapping{
-		Name:               "description",
+		Name:               resource.SEARCH_FIELD_DESCRIPTION,
 		Type:               "text",
 		Store:              true,
 		Index:              true,
@@ -41,27 +41,38 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
 	})
 
 	mapper.AddFieldMapping(&mapping.FieldMapping{
-		Name:               "tags",
+		Name:               resource.SEARCH_FIELD_TAGS,
 		Type:               "text",
 		Analyzer:           keyword.Name,
 		Store:              true,
 		Index:              true,
 		IncludeTermVectors: false,
-		IncludeInAll:       false,
+		IncludeInAll:       true,
 		DocValues:          false,
 	})
 
 	mapper.AddFieldMapping(&mapping.FieldMapping{
-		Name:               "folder",
+		Name:               resource.SEARCH_FIELD_FOLDER,
 		Type:               "text",
 		Analyzer:           keyword.Name,
 		Store:              true,
 		Index:              true,
 		IncludeTermVectors: false,
-		IncludeInAll:       false,
+		IncludeInAll:       true,
 		DocValues:          true, // will be needed for authz client
 	})
 
+	mapper.AddFieldMapping(&mapping.FieldMapping{
+		Name:               resource.SEARCH_FIELD_REPOSITORY,
+		Type:               "text",
+		Analyzer:           keyword.Name,
+		Store:              true,
+		Index:              true,
+		IncludeTermVectors: false,
+		IncludeInAll:       true,
+		DocValues:          true,
+	})
+
 	mapper.Dynamic = true
 
 	return mapper

pkg/storage/unified/search/bleve_test.go

@@ -7,16 +7,16 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
 )
 
 func TestBleveBackend(t *testing.T) {
-	t.Skip("flakey tests - skipping") // sort seems different in CI... sometimes!
-
 	dashboardskey := &resource.ResourceKey{
 		Namespace: "default",
 		Group:     "dashboard.grafana.app",
@@ -35,6 +35,9 @@ func TestBleveBackend(t *testing.T) {
 		FileThreshold: 5, // with more than 5 items we create a file on disk
 	}, tracing.NewNoopTracerService())
 
+	// AVOID NPE in test
+	resource.NewIndexMetrics(backend.opts.Root, backend)
+
 	rv := int64(10)
 	ctx := context.Background()
 	var dashboardsIndex resource.ResourceIndex
@@ -65,7 +68,7 @@ func TestBleveBackend(t *testing.T) {
 					Group:     "g",
 					Resource:  "dash",
 				},
-				Title:  "bbb (dash)",
+				Title:  "aaa (dash)",
 				Folder: "xxx",
 				Fields: map[string]any{
 					DASHBOARD_LEGACY_ID:    12,
@@ -82,7 +85,7 @@ func TestBleveBackend(t *testing.T) {
 					Group:     "g",
 					Resource:  "dash",
 				},
-				Title:  "aaa (dash)",
+				Title:  "bbb (dash)",
 				Folder: "xxx",
 				Fields: map[string]any{
 					DASHBOARD_LEGACY_ID:    12,
@@ -103,7 +106,10 @@ func TestBleveBackend(t *testing.T) {
 					Resource:  "dash",
 				},
 				Title:  "ccc (dash)",
-				Folder: "xxx",
+				Folder: "zzz",
+				RepoInfo: &utils.ResourceRepositoryInfo{
+					Name: "r0",
+				},
 				Fields: map[string]any{
 					DASHBOARD_LEGACY_ID: 12,
 				},
@@ -124,7 +130,7 @@ func TestBleveBackend(t *testing.T) {
 			},
 			Limit: 100000,
 			SortBy: []*resource.ResourceSearchRequest_Sort{
-				{Field: "title", Desc: true}, // ccc,bbb,aaa
+				{Field: resource.SEARCH_FIELD_TITLE, Desc: true}, // ccc,bbb,aaa
 			},
 			Facet: map[string]*resource.ResourceSearchRequest_Facet{
 				"tags": {
@@ -138,8 +144,11 @@ func TestBleveBackend(t *testing.T) {
 		require.NotNil(t, rsp.Results)
 		require.NotNil(t, rsp.Facet)
 
-		// Match the results
-		resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-dashboard.json"), rsp.Results)
+		t.Run("x", func(t *testing.T) {
+			t.Skip("flakey tests - skipping") // sort seems different in CI... sometimes!
+			// Match the results
+			resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-dashboard.json"), rsp.Results)
+		})
 
 		// Get the tags facets
 		facet, ok := rsp.Facet["tags"]
@@ -161,6 +170,12 @@ func TestBleveBackend(t *testing.T) {
 				}
 			]
 		}`, string(disp))
+
+		count, _ := index.DocCount(ctx, "")
+		assert.Equal(t, int64(3), count)
+
+		count, _ = index.DocCount(ctx, "zzz")
+		assert.Equal(t, int64(1), count)
 	})
 
 	t.Run("build folders", func(t *testing.T) {
@@ -216,6 +231,8 @@ func TestBleveBackend(t *testing.T) {
 	})
 
 	t.Run("simple federation", func(t *testing.T) {
+		t.Skip("flakey tests - skipping") // sort seems different in CI... sometimes!
+
 		// The other tests must run first to build the indexes
 		require.NotNil(t, dashboardsIndex)
 		require.NotNil(t, foldersIndex)

pkg/storage/unified/search/testdata/manual-dashboard.json

@@ -73,7 +73,7 @@
         [
           "aa"
         ],
-        "xxx",
+        "zzz",
         3,
         0,
         null,
@@ -92,8 +92,32 @@
     },
     {
       "cells": [
-        "ns/g/dash/aaa",
+        "ns/g/dash/bbb",
         "bbb (dash)",
+        [
+          "aa"
+        ],
+        "xxx",
+        2,
+        0,
+        null,
+        null,
+        null
+      ],
+      "object": {
+        "kind": "dash",
+        "apiVersion": "g",
+        "metadata": {
+          "name": "bbb",
+          "namespace": "ns",
+          "creationTimestamp": null
+        }
+      }
+    },
+    {
+      "cells": [
+        "ns/g/dash/aaa",
+        "aaa (dash)",
         [
           "aa",
           "bb"
@@ -114,30 +138,6 @@
           "creationTimestamp": null
         }
       }
-    },
-    {
-      "cells": [
-        "ns/g/dash/bbb",
-        "aaa (dash)",
-        [
-          "aa"
-        ],
-        "xxx",
-        2,
-        0,
-        null,
-        null,
-        null
-      ],
-      "object": {
-        "kind": "dash",
-        "apiVersion": "g",
-        "metadata": {
-          "name": "bbb",
-          "namespace": "ns",
-          "creationTimestamp": null
-        }
-      }
     }
   ]
 }

pkg/storage/unified/sql/data/resource_stats.sql

@@ -5,10 +5,13 @@ SELECT
   COUNT(*),
   MAX({{ .Ident "resource_version" }})
 FROM {{ .Ident "resource" }}
-{{ if .Namespace }}
 WHERE 1 = 1
+{{ if .Namespace }}
   AND {{ .Ident "namespace" }} = {{ .Arg .Namespace }}
 {{ end}}
+{{ if .Folder }}
+  AND {{ .Ident "folder" }} = {{ .Arg .Folder }}
+{{ end}}
 GROUP BY 
   {{ .Ident "namespace" }},
   {{ .Ident "group"     }},

pkg/storage/unified/sql/queries.go

@@ -75,11 +75,15 @@ func (r sqlResourceRequest) Validate() error {
 type sqlStatsRequest struct {
 	sqltemplate.SQLTemplate
 	Namespace string
+	Folder    string
 	MinCount  int
 }
 
 func (r sqlStatsRequest) Validate() error {
-	return nil // TODO
+	if r.Folder != "" && r.Namespace == "" {
+		return fmt.Errorf("folder constraint requires a namespace")
+	}
+	return nil
 }
 
 type historyPollResponse struct {

pkg/storage/unified/sql/queries_test.go

@@ -236,6 +236,15 @@ func TestUnifiedStorageQueries(t *testing.T) {
 						MinCount:    10, // Not yet used in query (only response filter)
 					},
 				},
+				{
+					Name: "query-folder",
+					Data: &sqlStatsRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						Namespace:   "default",
+						Folder:      "folder",
+						MinCount:    10, // Not yet used in query (only response filter)
+					},
+				},
 			},
 		}})
 }

pkg/storage/unified/sql/search.go

@@ -0,0 +1,83 @@
+package sql
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/storage/unified/resource"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/db"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/dbutil"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
+)
+
+// Support using SQL as fallback when the indexer is not running
+var _ resource.ResourceIndexServer = &backend{}
+
+// GetStats implements resource.ResourceIndexServer.
+// This will use the SQL index to count values
+func (b *backend) GetStats(ctx context.Context, req *resource.ResourceStatsRequest) (*resource.ResourceStatsResponse, error) {
+	ctx, span := b.tracer.Start(ctx, tracePrefix+".GetStats")
+	defer span.End()
+
+	sreq := &sqlStatsRequest{
+		SQLTemplate: sqltemplate.New(b.dialect),
+		Namespace:   req.Namespace,
+		Folder:      req.Folder,
+	}
+
+	rsp := &resource.ResourceStatsResponse{}
+	err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
+		rows, err := dbutil.QueryRows(ctx, tx, sqlResourceStats, sreq)
+		if err != nil {
+			return err
+		}
+		for rows.Next() {
+			row := resource.ResourceStats{}
+			err = rows.Scan(&row.Namespace, &row.Group, &row.Resource, &row.Count, &row.ResourceVersion)
+			if err != nil {
+				return err
+			}
+
+			rsp.Stats = append(rsp.Stats, &resource.ResourceStatsResponse_Stats{
+				Group:    row.Group,
+				Resource: row.Resource,
+				Count:    row.Count,
+			})
+		}
+		return err
+	})
+	if err != nil {
+		rsp.Error = resource.AsErrorResult(err)
+	}
+	return rsp, nil
+}
+
+// History implements resource.ResourceIndexServer.
+func (b *backend) History(context.Context, *resource.HistoryRequest) (*resource.HistoryResponse, error) {
+	return &resource.HistoryResponse{
+		Error: &resource.ErrorResult{
+			Code:    http.StatusNotImplemented,
+			Message: "SQL backend does not implement History",
+		},
+	}, nil
+}
+
+// Origin implements resource.ResourceIndexServer.
+func (b *backend) Origin(context.Context, *resource.OriginRequest) (*resource.OriginResponse, error) {
+	return &resource.OriginResponse{
+		Error: &resource.ErrorResult{
+			Code:    http.StatusNotImplemented,
+			Message: "SQL backend does not implement Origin",
+		},
+	}, nil
+}
+
+// Search implements resource.ResourceIndexServer.
+func (b *backend) Search(context.Context, *resource.ResourceSearchRequest) (*resource.ResourceSearchResponse, error) {
+	return &resource.ResourceSearchResponse{
+		Error: &resource.ErrorResult{
+			Code:    http.StatusNotImplemented,
+			Message: "SQL backend does not implement Search",
+		},
+	}, nil
+}

pkg/storage/unified/sql/testdata/mysql--resource_stats-query-folder.sql

@@ -0,0 +1,15 @@
+SELECT
+  `namespace`,
+  `group`,
+  `resource`,
+  COUNT(*),
+  MAX(`resource_version`)
+FROM `resource`
+WHERE 1 = 1
+  AND `namespace` = 'default'
+  AND `folder` = 'folder'
+GROUP BY 
+  `namespace`,
+  `group`,
+  `resource`
+;

pkg/storage/unified/sql/testdata/mysql--resource_stats-query.sql

@@ -5,6 +5,7 @@ SELECT
   COUNT(*),
   MAX(`resource_version`)
 FROM `resource`
+WHERE 1 = 1
 GROUP BY 
   `namespace`,
   `group`,

pkg/storage/unified/sql/testdata/postgres--resource_stats-query-folder.sql

@@ -0,0 +1,15 @@
+SELECT
+  "namespace",
+  "group",
+  "resource",
+  COUNT(*),
+  MAX("resource_version")
+FROM "resource"
+WHERE 1 = 1
+  AND "namespace" = 'default'
+  AND "folder" = 'folder'
+GROUP BY 
+  "namespace",
+  "group",
+  "resource"
+;

pkg/storage/unified/sql/testdata/postgres--resource_stats-query.sql

@@ -5,6 +5,7 @@ SELECT
   COUNT(*),
   MAX("resource_version")
 FROM "resource"
+WHERE 1 = 1
 GROUP BY 
   "namespace",
   "group",

pkg/storage/unified/sql/testdata/sqlite--resource_stats-query-folder.sql

@@ -0,0 +1,15 @@
+SELECT
+  "namespace",
+  "group",
+  "resource",
+  COUNT(*),
+  MAX("resource_version")
+FROM "resource"
+WHERE 1 = 1
+  AND "namespace" = 'default'
+  AND "folder" = 'folder'
+GROUP BY 
+  "namespace",
+  "group",
+  "resource"
+;

pkg/storage/unified/sql/testdata/sqlite--resource_stats-query.sql

@@ -5,6 +5,7 @@ SELECT
   COUNT(*),
   MAX("resource_version")
 FROM "resource"
+WHERE 1 = 1
 GROUP BY 
   "namespace",
   "group",