public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Search: Return counts for values within an folder/repository (#97534)

Browse Source
This commit is contained in:
Ryan McKinley
2024-12-10 20:37:37 +02:00
committed by GitHub
parent b05d60e5b5
commit ea17b79c09
27 changed files with 1195 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/resource/resource.pb.go
+680 -462
View File
File diff suppressed because it is too large Load Diff
pkg/storage/unified/resource/resource.proto
+35
View File
@@ -323,6 +323,38 @@ message WatchEvent {
Resource previous = 4;
}
// Get statistics across multiple resources
// For these queries, we do not need authorization to see the actual values
message ResourceStatsRequest {
// Namespace (tenant)
string namespace = 1;
// An optional list of group/resource identifiers
// when empty, we assume searching across everything
// NOTE, this query may need to federate across a few storage instances
repeated string kinds = 2;
// Limit the stats within a folder (not recursive!)
string folder = 3;
}
message ResourceStatsResponse {
message Stats {
// Resource group
string group = 1;
// Resource name
string resource = 2;
// Number of items
int64 count = 3;
}
// Error details
ErrorResult error = 1;
// All results exist within this key
repeated Stats stats = 2;
}
// Search within a single resource
message ResourceSearchRequest {
message Sort {
@@ -715,6 +747,9 @@ service ResourceStore {
service ResourceIndex {
rpc Search(ResourceSearchRequest) returns (ResourceSearchResponse);
// Get the resource stats
rpc GetStats(ResourceStatsRequest) returns (ResourceStatsResponse);
// Show resource history (and trash)
rpc History(HistoryRequest) returns (HistoryResponse);
pkg/storage/unified/resource/resource_grpc.pb.go
+43 -3
View File
@@ -348,9 +348,10 @@ var ResourceStore_ServiceDesc = grpc.ServiceDesc{
}
const (
ResourceIndex_Search_FullMethodName = "/resource.ResourceIndex/Search"
ResourceIndex_History_FullMethodName = "/resource.ResourceIndex/History"
ResourceIndex_Origin_FullMethodName = "/resource.ResourceIndex/Origin"
ResourceIndex_Search_FullMethodName = "/resource.ResourceIndex/Search"
ResourceIndex_GetStats_FullMethodName = "/resource.ResourceIndex/GetStats"
ResourceIndex_History_FullMethodName = "/resource.ResourceIndex/History"
ResourceIndex_Origin_FullMethodName = "/resource.ResourceIndex/Origin"
)
// ResourceIndexClient is the client API for ResourceIndex service.
@@ -361,6 +362,8 @@ const (
// It should be implemented with efficient indexes and does not need read-after-write semantics
type ResourceIndexClient interface {
Search(ctx context.Context, in *ResourceSearchRequest, opts ...grpc.CallOption) (*ResourceSearchResponse, error)
// Get the resource stats
GetStats(ctx context.Context, in *ResourceStatsRequest, opts ...grpc.CallOption) (*ResourceStatsResponse, error)
// Show resource history (and trash)
History(ctx context.Context, in *HistoryRequest, opts ...grpc.CallOption) (*HistoryResponse, error)
// Used for efficient provisioning
@@ -385,6 +388,16 @@ func (c *resourceIndexClient) Search(ctx context.Context, in *ResourceSearchRequ
return out, nil
}
func (c *resourceIndexClient) GetStats(ctx context.Context, in *ResourceStatsRequest, opts ...grpc.CallOption) (*ResourceStatsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ResourceStatsResponse)
err := c.cc.Invoke(ctx, ResourceIndex_GetStats_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *resourceIndexClient) History(ctx context.Context, in *HistoryRequest, opts ...grpc.CallOption) (*HistoryResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(HistoryResponse)
@@ -413,6 +426,8 @@ func (c *resourceIndexClient) Origin(ctx context.Context, in *OriginRequest, opt
// It should be implemented with efficient indexes and does not need read-after-write semantics
type ResourceIndexServer interface {
Search(context.Context, *ResourceSearchRequest) (*ResourceSearchResponse, error)
// Get the resource stats
GetStats(context.Context, *ResourceStatsRequest) (*ResourceStatsResponse, error)
// Show resource history (and trash)
History(context.Context, *HistoryRequest) (*HistoryResponse, error)
// Used for efficient provisioning
@@ -426,6 +441,9 @@ type UnimplementedResourceIndexServer struct {
func (UnimplementedResourceIndexServer) Search(context.Context, *ResourceSearchRequest) (*ResourceSearchResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Search not implemented")
}
func (UnimplementedResourceIndexServer) GetStats(context.Context, *ResourceStatsRequest) (*ResourceStatsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetStats not implemented")
}
func (UnimplementedResourceIndexServer) History(context.Context, *HistoryRequest) (*HistoryResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method History not implemented")
}
@@ -462,6 +480,24 @@ func _ResourceIndex_Search_Handler(srv interface{}, ctx context.Context, dec fun
return interceptor(ctx, in, info, handler)
}
func _ResourceIndex_GetStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ResourceStatsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ResourceIndexServer).GetStats(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ResourceIndex_GetStats_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ResourceIndexServer).GetStats(ctx, req.(*ResourceStatsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ResourceIndex_History_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(HistoryRequest)
if err := dec(in); err != nil {
@@ -509,6 +545,10 @@ var ResourceIndex_ServiceDesc = grpc.ServiceDesc{
MethodName: "Search",
Handler: _ResourceIndex_Search_Handler,
},
{
MethodName: "GetStats",
Handler: _ResourceIndex_GetStats_Handler,
},
{
MethodName: "History",
Handler: _ResourceIndex_History_Handler,
pkg/storage/unified/resource/search.go
+84 -2
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"slices"
"strings"
"sync"
"time"
@@ -47,7 +48,7 @@ type ResourceIndex interface {
Origin(ctx context.Context, req *OriginRequest) (*OriginResponse, error)
// Get the number of documents in the index
DocCount() (int, error)
DocCount(ctx context.Context, folder string) (int64, error)
}
// SearchBackend contains the technology specific logic to support search
@@ -168,6 +169,87 @@ func (s *searchSupport) Search(ctx context.Context, req *ResourceSearchRequest)
return idx.Search(ctx, s.access, req, federate)
}
// GetStats implements ResourceServer.
func (s *searchSupport) GetStats(ctx context.Context, req *ResourceStatsRequest) (*ResourceStatsResponse, error) {
if req.Namespace == "" {
return &ResourceStatsResponse{
Error: NewBadRequestError("missing namespace"),
}, nil
}
rsp := &ResourceStatsResponse{}
// Explicit list of kinds
if len(req.Kinds) > 0 {
rsp.Stats = make([]*ResourceStatsResponse_Stats, len(req.Kinds))
for i, k := range req.Kinds {
parts := strings.SplitN(k, "/", 2)
index, err := s.getOrCreateIndex(ctx, NamespacedResource{
Namespace: req.Namespace,
Group: parts[0],
Resource: parts[1],
})
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
}
count, err := index.DocCount(ctx, req.Folder)
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
}
rsp.Stats[i] = &ResourceStatsResponse_Stats{
Group: parts[0],
Resource: parts[1],
Count: count,
}
}
return rsp, nil
}
stats, err := s.storage.GetResourceStats(ctx, req.Namespace, 0)
if err != nil {
return &ResourceStatsResponse{
Error: AsErrorResult(err),
}, nil
}
rsp.Stats = make([]*ResourceStatsResponse_Stats, len(stats))
// When not filtered by folder or repository, we can use the results directly
if req.Folder == "" {
for i, stat := range stats {
rsp.Stats[i] = &ResourceStatsResponse_Stats{
Group: stat.Group,
Resource: stat.Resource,
Count: stat.Count,
}
}
return rsp, nil
}
for i, stat := range stats {
index, err := s.getOrCreateIndex(ctx, NamespacedResource{
Namespace: req.Namespace,
Group: stat.Group,
Resource: stat.Resource,
})
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
}
count, err := index.DocCount(ctx, req.Folder)
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
}
rsp.Stats[i] = &ResourceStatsResponse_Stats{
Group: stat.Group,
Resource: stat.Resource,
Count: count,
}
}
return rsp, nil
}
// init is called during startup. any failure will block startup and continued execution
func (s *searchSupport) init(ctx context.Context) error {
ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Init")
@@ -360,7 +442,7 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
}
// Record the number of objects indexed for the kind/resource
docCount, err := index.DocCount()
docCount, err := index.DocCount(ctx, "")
if err != nil {
s.log.Warn("error getting doc count", "error", err)
}
pkg/storage/unified/resource/server.go
+16
View File
@@ -920,6 +920,22 @@ func (s *server) Search(ctx context.Context, req *ResourceSearchRequest) (*Resou
return s.search.Search(ctx, req)
}
// GetStats implements ResourceServer.
func (s *server) GetStats(ctx context.Context, req *ResourceStatsRequest) (*ResourceStatsResponse, error) {
if err := s.Init(ctx); err != nil {
return nil, err
}
if s.search == nil {
// If the backend implements "GetStats", we can use it
srv, ok := s.backend.(ResourceIndexServer)
if ok {
return srv.GetStats(ctx, req)
}
return nil, fmt.Errorf("search index not configured")
}
return s.search.GetStats(ctx, req)
}
// History implements ResourceServer.
func (s *server) History(ctx context.Context, req *HistoryRequest) (*HistoryResponse, error) {
return s.search.History(ctx, req)