Search: Rename managed object index (#101856)

This commit is contained in:
Ryan McKinley
2025-03-10 19:48:53 +03:00
committed by GitHub
parent 9858e40a02
commit 64f1df69d5
22 changed files with 688 additions and 635 deletions
+4 -4
View File
@@ -52,12 +52,12 @@ func (d *directResourceClient) List(ctx context.Context, in *resource.ListReques
return d.server.List(ctx, in)
}
func (d *directResourceClient) ListRepositoryObjects(ctx context.Context, in *resource.ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*resource.ListRepositoryObjectsResponse, error) {
return d.server.ListRepositoryObjects(ctx, in)
func (d *directResourceClient) ListManagedObjects(ctx context.Context, in *resource.ListManagedObjectsRequest, opts ...grpc.CallOption) (*resource.ListManagedObjectsResponse, error) {
return d.server.ListManagedObjects(ctx, in)
}
func (d *directResourceClient) CountRepositoryObjects(ctx context.Context, in *resource.CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*resource.CountRepositoryObjectsResponse, error) {
return d.server.CountRepositoryObjects(ctx, in)
func (d *directResourceClient) CountManagedObjects(ctx context.Context, in *resource.CountManagedObjectsRequest, opts ...grpc.CallOption) (*resource.CountManagedObjectsResponse, error) {
return d.server.CountManagedObjects(ctx, in)
}
// PutBlob implements ResourceClient.
@@ -262,11 +262,11 @@ func (a *dashboardSqlAccess) Search(ctx context.Context, req *resource.ResourceS
return a.dashboardSearchClient.Search(ctx, req)
}
func (a *dashboardSqlAccess) ListRepositoryObjects(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
func (a *dashboardSqlAccess) ListManagedObjects(ctx context.Context, req *resource.ListManagedObjectsRequest) (*resource.ListManagedObjectsResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (a *dashboardSqlAccess) CountRepositoryObjects(context.Context, *resource.CountRepositoryObjectsRequest) (*resource.CountRepositoryObjectsResponse, error) {
func (a *dashboardSqlAccess) CountManagedObjects(context.Context, *resource.CountManagedObjectsRequest) (*resource.CountManagedObjectsResponse, error) {
return nil, fmt.Errorf("not implemented")
}
+2 -2
View File
@@ -674,7 +674,7 @@ func (m *MockClient) Search(ctx context.Context, in *resource.ResourceSearchRequ
func (m *MockClient) GetStats(ctx context.Context, in *resource.ResourceStatsRequest, opts ...grpc.CallOption) (*resource.ResourceStatsResponse, error) {
return nil, nil
}
func (m *MockClient) CountRepositoryObjects(ctx context.Context, in *resource.CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*resource.CountRepositoryObjectsResponse, error) {
func (m *MockClient) CountManagedObjects(ctx context.Context, in *resource.CountManagedObjectsRequest, opts ...grpc.CallOption) (*resource.CountManagedObjectsResponse, error) {
return nil, nil
}
func (m *MockClient) Watch(ctx context.Context, in *resource.WatchRequest, opts ...grpc.CallOption) (resource.ResourceStore_WatchClient, error) {
@@ -704,7 +704,7 @@ func (m *MockClient) PutBlob(ctx context.Context, in *resource.PutBlobRequest, o
func (m *MockClient) List(ctx context.Context, in *resource.ListRequest, opts ...grpc.CallOption) (*resource.ListResponse, error) {
return nil, nil
}
func (m *MockClient) ListRepositoryObjects(ctx context.Context, in *resource.ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*resource.ListRepositoryObjectsResponse, error) {
func (m *MockClient) ListManagedObjects(ctx context.Context, in *resource.ListManagedObjectsRequest, opts ...grpc.CallOption) (*resource.ListManagedObjectsResponse, error) {
return nil, nil
}
func (m *MockClient) IsHealthy(ctx context.Context, in *resource.HealthCheckRequest, opts ...grpc.CallOption) (*resource.HealthCheckResponse, error) {
+1 -1
View File
@@ -155,7 +155,7 @@ func TestDeleteWithSuggestionAndConflict(t *testing.T) {
type resourceClientMock struct {
resource.ResourceStoreClient
resource.ResourceIndexClient
resource.RepositoryIndexClient
resource.ManagedObjectIndexClient
resource.BulkStoreClient
resource.BlobStoreClient
resource.DiagnosticsClient
+21 -21
View File
@@ -24,7 +24,7 @@ import (
type ResourceClient interface {
ResourceStoreClient
ResourceIndexClient
RepositoryIndexClient
ManagedObjectIndexClient
BulkStoreClient
BlobStoreClient
DiagnosticsClient
@@ -34,7 +34,7 @@ type ResourceClient interface {
type resourceClient struct {
ResourceStoreClient
ResourceIndexClient
RepositoryIndexClient
ManagedObjectIndexClient
BulkStoreClient
BlobStoreClient
DiagnosticsClient
@@ -43,12 +43,12 @@ type resourceClient struct {
func NewLegacyResourceClient(channel *grpc.ClientConn) ResourceClient {
cc := grpchan.InterceptClientConn(channel, grpcUtils.UnaryClientInterceptor, grpcUtils.StreamClientInterceptor)
return &resourceClient{
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
RepositoryIndexClient: NewRepositoryIndexClient(cc),
BulkStoreClient: NewBulkStoreClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
ManagedObjectIndexClient: NewManagedObjectIndexClient(cc),
BulkStoreClient: NewBulkStoreClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
}
}
@@ -60,7 +60,7 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient {
for _, desc := range []*grpc.ServiceDesc{
&ResourceStore_ServiceDesc,
&ResourceIndex_ServiceDesc,
&RepositoryIndex_ServiceDesc,
&ManagedObjectIndex_ServiceDesc,
&BlobStore_ServiceDesc,
&BulkStore_ServiceDesc,
&Diagnostics_ServiceDesc,
@@ -82,12 +82,12 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient {
cc := grpchan.InterceptClientConn(channel, clientInt.UnaryClientInterceptor, clientInt.StreamClientInterceptor)
return &resourceClient{
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
RepositoryIndexClient: NewRepositoryIndexClient(cc),
BulkStoreClient: NewBulkStoreClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
ManagedObjectIndexClient: NewManagedObjectIndexClient(cc),
BulkStoreClient: NewBulkStoreClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
}
}
@@ -124,12 +124,12 @@ func NewRemoteResourceClient(tracer tracing.Tracer, conn *grpc.ClientConn, cfg R
cc := grpchan.InterceptClientConn(conn, clientInt.UnaryClientInterceptor, clientInt.StreamClientInterceptor)
return &resourceClient{
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
BulkStoreClient: NewBulkStoreClient(cc),
RepositoryIndexClient: NewRepositoryIndexClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
ResourceStoreClient: NewResourceStoreClient(cc),
ResourceIndexClient: NewResourceIndexClient(cc),
BlobStoreClient: NewBlobStoreClient(cc),
BulkStoreClient: NewBulkStoreClient(cc),
ManagedObjectIndexClient: NewManagedObjectIndexClient(cc),
DiagnosticsClient: NewDiagnosticsClient(cc),
}, nil
}
+23 -11
View File
@@ -107,10 +107,22 @@ type IndexableDocument struct {
// When the resource is managed by an upstream repository
Manager *utils.ManagerProperties `json:"manager,omitempty"`
// indexed only field for faceting manager info
ManagedBy string `json:"managedBy,omitempty"`
// When the manager knows about file paths
Source *utils.SourceProperties `json:"source,omitempty"`
}
func (m *IndexableDocument) UpdateCopyFields() *IndexableDocument {
m.TitleNgram = m.Title
m.TitlePhrase = strings.ToLower(m.Title) // Lowercase for case-insensitive sorting ?? in the analyzer?
if m.Manager != nil {
m.ManagedBy = fmt.Sprintf("%s:%s", m.Manager.Kind, m.Manager.Identity)
}
return m
}
func (m *IndexableDocument) Type() string {
return m.Key.Resource
}
@@ -169,20 +181,19 @@ func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAcces
}
}
doc := &IndexableDocument{
Key: key,
RV: rv,
Name: key.Name,
Title: title, // We always want *something* to display
TitleNgram: title,
TitlePhrase: strings.ToLower(title), // Lowercase for case-insensitive sorting
Labels: obj.GetLabels(),
Folder: obj.GetFolder(),
CreatedBy: obj.GetCreatedBy(),
UpdatedBy: obj.GetUpdatedBy(),
Key: key,
RV: rv,
Name: key.Name,
Title: title, // We always want *something* to display
Labels: obj.GetLabels(),
Folder: obj.GetFolder(),
CreatedBy: obj.GetCreatedBy(),
UpdatedBy: obj.GetUpdatedBy(),
}
m, ok := obj.GetManagerProperties()
if ok {
doc.Manager = &m
doc.ManagedBy = fmt.Sprintf("%s:%s", m.Kind, m.Identity)
}
s, ok := obj.GetSourceProperties()
if ok {
@@ -196,7 +207,7 @@ func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAcces
if err != nil && tt != nil {
doc.Updated = tt.UnixMilli()
}
return doc
return doc.UpdateCopyFields()
}
func StandardDocumentBuilder() DocumentBuilder {
@@ -280,6 +291,7 @@ const SEARCH_FIELD_CREATED_BY = "createdBy"
const SEARCH_FIELD_UPDATED = "updated"
const SEARCH_FIELD_UPDATED_BY = "updatedBy"
const SEARCH_FIELD_MANAGED_BY = "managedBy" // {kind}:{id}
const SEARCH_FIELD_MANAGER_KIND = "manager.kind"
const SEARCH_FIELD_MANAGER_ID = "manager.id"
const SEARCH_FIELD_SOURCE_PATH = "source.path"
@@ -35,9 +35,9 @@ func TestStandardDocumentBuilder(t *testing.T) {
},
"name": "test1",
"rv": 10,
"title": "test playlist unified storage",
"title_phrase": "test playlist unified storage",
"title_ngram": "test playlist unified storage",
"title": "Test Playlist from Unified Storage",
"title_ngram": "Test Playlist from Unified Storage",
"title_phrase": "test playlist from unified storage",
"created": 1717236672000,
"createdBy": "user:ABC",
"updatedBy": "user:XYZ",
@@ -45,6 +45,7 @@ func TestStandardDocumentBuilder(t *testing.T) {
"kind": "repo",
"id": "something"
},
"managedBy": "repo:something",
"source": {
"path": "path/in/system.json",
"checksum": "xyz"
File diff suppressed because it is too large Load Diff
+23 -17
View File
@@ -503,18 +503,21 @@ message ResourceSearchResponse {
// List items within a resource type & repository name
// Access control is managed above this request
message ListRepositoryObjectsRequest {
message ListManagedObjectsRequest {
// Starting from the requested page (other query parameters must match!)
string next_page_token = 1;
// Namespace (tenant)
string namespace = 2;
// The name of the repository
string name = 3;
// The manager type (eg, terraform vs repo)
string kind = 3;
// The name of the manager
string id = 4;
}
message ListRepositoryObjectsResponse {
message ListManagedObjectsResponse {
message Item {
// The resource object key
ResourceKey object = 1;
@@ -546,22 +549,25 @@ message ListRepositoryObjectsResponse {
}
// Count the items that exist with
message CountRepositoryObjectsRequest {
message CountManagedObjectsRequest {
// Namespace (tenant)
string namespace = 1;
// The name of the repository
// empty to count across all repositories
string name = 2;
// Manager kind: terraform, plugin, kubectl, repo
string kind = 2;
// Name of the manager (meaningful inside kind)
string id = 3;
}
// Count the items that exist with
message CountRepositoryObjectsResponse {
message CountManagedObjectsResponse {
message ResourceCount {
string repository = 1;
string group = 2;
string resource = 3;
int64 count = 4;
string kind = 1;
string id = 2;
string group = 3;
string resource = 4;
int64 count = 5;
}
// Resource counts
@@ -834,14 +840,14 @@ service ResourceIndex {
rpc GetStats(ResourceStatsRequest) returns (ResourceStatsResponse);
}
// Query repository info from the search index.
// Query managed objects
// Results access control is based on access to the repository *not* the items
service RepositoryIndex {
service ManagedObjectIndex {
// Describe how many resources of each type exist within a repository
rpc CountRepositoryObjects(CountRepositoryObjectsRequest) returns (CountRepositoryObjectsResponse);
rpc CountManagedObjects(CountManagedObjectsRequest) returns (CountManagedObjectsResponse);
// List the resources of a specific kind within a repository
rpc ListRepositoryObjects(ListRepositoryObjectsRequest) returns (ListRepositoryObjectsResponse);
rpc ListManagedObjects(ListManagedObjectsRequest) returns (ListManagedObjectsResponse);
}
service BlobStore {
@@ -650,136 +650,136 @@ var ResourceIndex_ServiceDesc = grpc.ServiceDesc{
}
const (
RepositoryIndex_CountRepositoryObjects_FullMethodName = "/resource.RepositoryIndex/CountRepositoryObjects"
RepositoryIndex_ListRepositoryObjects_FullMethodName = "/resource.RepositoryIndex/ListRepositoryObjects"
ManagedObjectIndex_CountManagedObjects_FullMethodName = "/resource.ManagedObjectIndex/CountManagedObjects"
ManagedObjectIndex_ListManagedObjects_FullMethodName = "/resource.ManagedObjectIndex/ListManagedObjects"
)
// RepositoryIndexClient is the client API for RepositoryIndex service.
// ManagedObjectIndexClient is the client API for ManagedObjectIndex service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// Query repository info from the search index.
// Query managed objects
// Results access control is based on access to the repository *not* the items
type RepositoryIndexClient interface {
type ManagedObjectIndexClient interface {
// Describe how many resources of each type exist within a repository
CountRepositoryObjects(ctx context.Context, in *CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*CountRepositoryObjectsResponse, error)
CountManagedObjects(ctx context.Context, in *CountManagedObjectsRequest, opts ...grpc.CallOption) (*CountManagedObjectsResponse, error)
// List the resources of a specific kind within a repository
ListRepositoryObjects(ctx context.Context, in *ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*ListRepositoryObjectsResponse, error)
ListManagedObjects(ctx context.Context, in *ListManagedObjectsRequest, opts ...grpc.CallOption) (*ListManagedObjectsResponse, error)
}
type repositoryIndexClient struct {
type managedObjectIndexClient struct {
cc grpc.ClientConnInterface
}
func NewRepositoryIndexClient(cc grpc.ClientConnInterface) RepositoryIndexClient {
return &repositoryIndexClient{cc}
func NewManagedObjectIndexClient(cc grpc.ClientConnInterface) ManagedObjectIndexClient {
return &managedObjectIndexClient{cc}
}
func (c *repositoryIndexClient) CountRepositoryObjects(ctx context.Context, in *CountRepositoryObjectsRequest, opts ...grpc.CallOption) (*CountRepositoryObjectsResponse, error) {
func (c *managedObjectIndexClient) CountManagedObjects(ctx context.Context, in *CountManagedObjectsRequest, opts ...grpc.CallOption) (*CountManagedObjectsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(CountRepositoryObjectsResponse)
err := c.cc.Invoke(ctx, RepositoryIndex_CountRepositoryObjects_FullMethodName, in, out, cOpts...)
out := new(CountManagedObjectsResponse)
err := c.cc.Invoke(ctx, ManagedObjectIndex_CountManagedObjects_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *repositoryIndexClient) ListRepositoryObjects(ctx context.Context, in *ListRepositoryObjectsRequest, opts ...grpc.CallOption) (*ListRepositoryObjectsResponse, error) {
func (c *managedObjectIndexClient) ListManagedObjects(ctx context.Context, in *ListManagedObjectsRequest, opts ...grpc.CallOption) (*ListManagedObjectsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListRepositoryObjectsResponse)
err := c.cc.Invoke(ctx, RepositoryIndex_ListRepositoryObjects_FullMethodName, in, out, cOpts...)
out := new(ListManagedObjectsResponse)
err := c.cc.Invoke(ctx, ManagedObjectIndex_ListManagedObjects_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// RepositoryIndexServer is the server API for RepositoryIndex service.
// All implementations should embed UnimplementedRepositoryIndexServer
// ManagedObjectIndexServer is the server API for ManagedObjectIndex service.
// All implementations should embed UnimplementedManagedObjectIndexServer
// for forward compatibility
//
// Query repository info from the search index.
// Query managed objects
// Results access control is based on access to the repository *not* the items
type RepositoryIndexServer interface {
type ManagedObjectIndexServer interface {
// Describe how many resources of each type exist within a repository
CountRepositoryObjects(context.Context, *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error)
CountManagedObjects(context.Context, *CountManagedObjectsRequest) (*CountManagedObjectsResponse, error)
// List the resources of a specific kind within a repository
ListRepositoryObjects(context.Context, *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error)
ListManagedObjects(context.Context, *ListManagedObjectsRequest) (*ListManagedObjectsResponse, error)
}
// UnimplementedRepositoryIndexServer should be embedded to have forward compatible implementations.
type UnimplementedRepositoryIndexServer struct {
// UnimplementedManagedObjectIndexServer should be embedded to have forward compatible implementations.
type UnimplementedManagedObjectIndexServer struct {
}
func (UnimplementedRepositoryIndexServer) CountRepositoryObjects(context.Context, *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CountRepositoryObjects not implemented")
func (UnimplementedManagedObjectIndexServer) CountManagedObjects(context.Context, *CountManagedObjectsRequest) (*CountManagedObjectsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CountManagedObjects not implemented")
}
func (UnimplementedRepositoryIndexServer) ListRepositoryObjects(context.Context, *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListRepositoryObjects not implemented")
func (UnimplementedManagedObjectIndexServer) ListManagedObjects(context.Context, *ListManagedObjectsRequest) (*ListManagedObjectsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListManagedObjects not implemented")
}
// UnsafeRepositoryIndexServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to RepositoryIndexServer will
// UnsafeManagedObjectIndexServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ManagedObjectIndexServer will
// result in compilation errors.
type UnsafeRepositoryIndexServer interface {
mustEmbedUnimplementedRepositoryIndexServer()
type UnsafeManagedObjectIndexServer interface {
mustEmbedUnimplementedManagedObjectIndexServer()
}
func RegisterRepositoryIndexServer(s grpc.ServiceRegistrar, srv RepositoryIndexServer) {
s.RegisterService(&RepositoryIndex_ServiceDesc, srv)
func RegisterManagedObjectIndexServer(s grpc.ServiceRegistrar, srv ManagedObjectIndexServer) {
s.RegisterService(&ManagedObjectIndex_ServiceDesc, srv)
}
func _RepositoryIndex_CountRepositoryObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CountRepositoryObjectsRequest)
func _ManagedObjectIndex_CountManagedObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CountManagedObjectsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RepositoryIndexServer).CountRepositoryObjects(ctx, in)
return srv.(ManagedObjectIndexServer).CountManagedObjects(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: RepositoryIndex_CountRepositoryObjects_FullMethodName,
FullMethod: ManagedObjectIndex_CountManagedObjects_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RepositoryIndexServer).CountRepositoryObjects(ctx, req.(*CountRepositoryObjectsRequest))
return srv.(ManagedObjectIndexServer).CountManagedObjects(ctx, req.(*CountManagedObjectsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _RepositoryIndex_ListRepositoryObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListRepositoryObjectsRequest)
func _ManagedObjectIndex_ListManagedObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListManagedObjectsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(RepositoryIndexServer).ListRepositoryObjects(ctx, in)
return srv.(ManagedObjectIndexServer).ListManagedObjects(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: RepositoryIndex_ListRepositoryObjects_FullMethodName,
FullMethod: ManagedObjectIndex_ListManagedObjects_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(RepositoryIndexServer).ListRepositoryObjects(ctx, req.(*ListRepositoryObjectsRequest))
return srv.(ManagedObjectIndexServer).ListManagedObjects(ctx, req.(*ListManagedObjectsRequest))
}
return interceptor(ctx, in, info, handler)
}
// RepositoryIndex_ServiceDesc is the grpc.ServiceDesc for RepositoryIndex service.
// ManagedObjectIndex_ServiceDesc is the grpc.ServiceDesc for ManagedObjectIndex service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var RepositoryIndex_ServiceDesc = grpc.ServiceDesc{
ServiceName: "resource.RepositoryIndex",
HandlerType: (*RepositoryIndexServer)(nil),
var ManagedObjectIndex_ServiceDesc = grpc.ServiceDesc{
ServiceName: "resource.ManagedObjectIndex",
HandlerType: (*ManagedObjectIndexServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "CountRepositoryObjects",
Handler: _RepositoryIndex_CountRepositoryObjects_Handler,
MethodName: "CountManagedObjects",
Handler: _ManagedObjectIndex_CountManagedObjects_Handler,
},
{
MethodName: "ListRepositoryObjects",
Handler: _RepositoryIndex_ListRepositoryObjects_Handler,
MethodName: "ListManagedObjects",
Handler: _ManagedObjectIndex_ListManagedObjects_Handler,
},
},
Streams: []grpc.StreamDesc{},
+18 -18
View File
@@ -48,10 +48,10 @@ type ResourceIndex interface {
Search(ctx context.Context, access types.AccessClient, req *ResourceSearchRequest, federate []ResourceIndex) (*ResourceSearchResponse, error)
// List within an response
ListRepositoryObjects(ctx context.Context, req *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error)
ListManagedObjects(ctx context.Context, req *ListManagedObjectsRequest) (*ListManagedObjectsResponse, error)
// Counts the values in a repo
CountRepositoryObjects(ctx context.Context) ([]*CountRepositoryObjectsResponse_ResourceCount, error)
CountManagedObjects(ctx context.Context) ([]*CountManagedObjectsResponse_ResourceCount, error)
// Get the number of documents in the index
DocCount(ctx context.Context, folder string) (int64, error)
@@ -99,8 +99,8 @@ type searchSupport struct {
}
var (
_ ResourceIndexServer = (*searchSupport)(nil)
_ RepositoryIndexServer = (*searchSupport)(nil)
_ ResourceIndexServer = (*searchSupport)(nil)
_ ManagedObjectIndexServer = (*searchSupport)(nil)
)
func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.AccessClient, blob BlobSupport, tracer trace.Tracer) (support *searchSupport, err error) {
@@ -139,14 +139,14 @@ func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.A
return support, err
}
func (s *searchSupport) ListRepositoryObjects(ctx context.Context, req *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error) {
func (s *searchSupport) ListManagedObjects(ctx context.Context, req *ListManagedObjectsRequest) (*ListManagedObjectsResponse, error) {
if req.NextPageToken != "" {
return &ListRepositoryObjectsResponse{
return &ListManagedObjectsResponse{
Error: NewBadRequestError("multiple pages not yet supported"),
}, nil
}
rsp := &ListRepositoryObjectsResponse{}
rsp := &ListManagedObjectsResponse{}
stats, err := s.storage.GetResourceStats(ctx, req.Namespace, 0)
if err != nil {
rsp.Error = AsErrorResult(err)
@@ -164,7 +164,7 @@ func (s *searchSupport) ListRepositoryObjects(ctx context.Context, req *ListRepo
return rsp, nil
}
kind, err := idx.ListRepositoryObjects(ctx, req)
kind, err := idx.ListManagedObjects(ctx, req)
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
@@ -179,15 +179,15 @@ func (s *searchSupport) ListRepositoryObjects(ctx context.Context, req *ListRepo
}
// Sort based on path
slices.SortFunc(rsp.Items, func(a, b *ListRepositoryObjectsResponse_Item) int {
slices.SortFunc(rsp.Items, func(a, b *ListManagedObjectsResponse_Item) int {
return cmp.Compare(a.Path, b.Path)
})
return rsp, nil
}
func (s *searchSupport) CountRepositoryObjects(ctx context.Context, req *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error) {
rsp := &CountRepositoryObjectsResponse{}
func (s *searchSupport) CountManagedObjects(ctx context.Context, req *CountManagedObjectsRequest) (*CountManagedObjectsResponse, error) {
rsp := &CountManagedObjectsResponse{}
stats, err := s.storage.GetResourceStats(ctx, req.Namespace, 0)
if err != nil {
rsp.Error = AsErrorResult(err)
@@ -205,27 +205,27 @@ func (s *searchSupport) CountRepositoryObjects(ctx context.Context, req *CountRe
return rsp, nil
}
counts, err := idx.CountRepositoryObjects(ctx)
counts, err := idx.CountManagedObjects(ctx)
if err != nil {
rsp.Error = AsErrorResult(err)
return rsp, nil
}
if req.Name == "" {
if req.Id == "" {
rsp.Items = append(rsp.Items, counts...)
} else {
for _, k := range counts {
if k.Repository == req.Name {
k.Repository = "" // avoid duplicate response metadata
if k.Id == req.Id {
rsp.Items = append(rsp.Items, k)
}
}
}
}
// Sort based on repo/group/resource
slices.SortFunc(rsp.Items, func(a, b *CountRepositoryObjectsResponse_ResourceCount) int {
// Sort based on manager/group/resource
slices.SortFunc(rsp.Items, func(a, b *CountManagedObjectsResponse_ResourceCount) int {
return cmp.Or(
cmp.Compare(a.Repository, b.Repository),
cmp.Compare(a.Kind, b.Kind),
cmp.Compare(a.Id, b.Id),
cmp.Compare(a.Group, b.Group),
cmp.Compare(a.Resource, b.Resource),
)
+5 -5
View File
@@ -28,7 +28,7 @@ type ResourceServer interface {
ResourceStoreServer
BulkStoreServer
ResourceIndexServer
RepositoryIndexServer
ManagedObjectIndexServer
BlobStoreServer
DiagnosticsServer
}
@@ -1105,12 +1105,12 @@ func (s *server) GetStats(ctx context.Context, req *ResourceStatsRequest) (*Reso
return s.search.GetStats(ctx, req)
}
func (s *server) ListRepositoryObjects(ctx context.Context, req *ListRepositoryObjectsRequest) (*ListRepositoryObjectsResponse, error) {
return s.search.ListRepositoryObjects(ctx, req)
func (s *server) ListManagedObjects(ctx context.Context, req *ListManagedObjectsRequest) (*ListManagedObjectsResponse, error) {
return s.search.ListManagedObjects(ctx, req)
}
func (s *server) CountRepositoryObjects(ctx context.Context, req *CountRepositoryObjectsRequest) (*CountRepositoryObjectsResponse, error) {
return s.search.CountRepositoryObjects(ctx, req)
func (s *server) CountManagedObjects(ctx context.Context, req *CountManagedObjectsRequest) (*CountManagedObjectsResponse, error) {
return s.search.CountManagedObjects(ctx, req)
}
// IsHealthy implements ResourceServer.
@@ -16,7 +16,7 @@
}
},
"spec": {
"title": "test playlist unified storage",
"title": "Test Playlist from Unified Storage",
"description": "description for the test playlist"
}
}
+38 -19
View File
@@ -263,6 +263,8 @@ type bleveIndex struct {
// Write implements resource.DocumentIndex.
func (b *bleveIndex) Write(v *resource.IndexableDocument) error {
v = v.UpdateCopyFields()
// remove references (for now!)
v.References = nil
if b.batch != nil {
@@ -297,21 +299,33 @@ func (b *bleveIndex) Flush() (err error) {
return err
}
func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resource.ListManagedObjectsRequest) (*resource.ListManagedObjectsResponse, error) {
if req.NextPageToken != "" {
return nil, fmt.Errorf("next page not implemented yet")
}
if req.Name == "" {
return &resource.ListRepositoryObjectsResponse{
Error: resource.NewBadRequestError("empty repository name"),
if req.Kind == "" {
return &resource.ListManagedObjectsResponse{
Error: resource.NewBadRequestError("empty manager kind"),
}, nil
}
if req.Id == "" {
return &resource.ListManagedObjectsResponse{
Error: resource.NewBadRequestError("empty manager id"),
}, nil
}
q := bleve.NewBooleanQuery()
q.AddMust(&query.TermQuery{
Term: req.Kind,
FieldVal: resource.SEARCH_FIELD_MANAGER_KIND,
})
q.AddMust(&query.TermQuery{
Term: req.Id,
FieldVal: resource.SEARCH_FIELD_MANAGER_ID,
})
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: &query.TermQuery{
Term: req.Name,
FieldVal: resource.SEARCH_FIELD_MANAGER_ID,
},
Query: q,
Fields: []string{
resource.SEARCH_FIELD_TITLE,
resource.SEARCH_FIELD_FOLDER,
@@ -366,9 +380,9 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
return 0
}
rsp := &resource.ListRepositoryObjectsResponse{}
rsp := &resource.ListManagedObjectsResponse{}
for _, hit := range found.Hits {
item := &resource.ListRepositoryObjectsResponse_Item{
item := &resource.ListManagedObjectsResponse_Item{
Object: &resource.ResourceKey{},
Hash: asString(hit.Fields[resource.SEARCH_FIELD_SOURCE_CHECKSUM]),
Path: asString(hit.Fields[resource.SEARCH_FIELD_SOURCE_PATH]),
@@ -385,27 +399,32 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
return rsp, nil
}
func (b *bleveIndex) CountRepositoryObjects(ctx context.Context) ([]*resource.CountRepositoryObjectsResponse_ResourceCount, error) {
func (b *bleveIndex) CountManagedObjects(ctx context.Context) ([]*resource.CountManagedObjectsResponse_ResourceCount, error) {
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: bleve.NewMatchAllQuery(),
Size: 0,
Facets: bleve.FacetsRequest{
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_MANAGER_ID, 1000), // typically less then 5
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_MANAGED_BY, 1000), // typically less then 5
},
})
if err != nil {
return nil, err
}
vals := make([]*resource.CountRepositoryObjectsResponse_ResourceCount, 0)
vals := make([]*resource.CountManagedObjectsResponse_ResourceCount, 0)
f, ok := found.Facets["count"]
if ok && f.Terms != nil {
for _, v := range f.Terms.Terms() {
vals = append(vals, &resource.CountRepositoryObjectsResponse_ResourceCount{
Repository: v.Term,
Group: b.key.Group,
Resource: b.key.Resource,
Count: int64(v.Count),
})
val := v.Term
idx := strings.Index(val, ":")
if idx > 0 {
vals = append(vals, &resource.CountManagedObjectsResponse_ResourceCount{
Kind: val[0:idx],
Id: val[idx+1:],
Group: b.key.Group,
Resource: b.key.Resource,
Count: int64(v.Count),
})
}
}
}
return vals, nil
+12 -2
View File
@@ -34,7 +34,7 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
// for searching by title - uses an edge ngram token filter
titleSearchMapping := bleve.NewTextFieldMapping()
titleSearchMapping.Analyzer = TITLE_ANALYZER
titleSearchMapping.Store = true
titleSearchMapping.Store = false // already stored in title
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_NGRAM, titleSearchMapping)
// mapping for title to search on words/tokens larger than the ngram size
@@ -45,6 +45,7 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
// for filtering/sorting by title full phrase
titlePhraseMapping := bleve.NewKeywordFieldMapping()
titleSearchMapping.Store = false // already stored in title
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_PHRASE, titlePhraseMapping)
descriptionMapping := &mapping.FieldMapping{
@@ -124,8 +125,17 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
})
source.AddFieldMappingsAt("timestampMillis", mapping.NewNumericFieldMapping())
mapper.AddSubDocumentMapping("manager", manager)
mapper.AddSubDocumentMapping("source", source)
mapper.AddSubDocumentMapping("manager", manager)
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_MANAGED_BY, &mapping.FieldMapping{
Name: "managedBy",
Type: "text",
Analyzer: keyword.Name,
Index: true, // only used for faceting
Store: false,
IncludeTermVectors: false,
IncludeInAll: false,
})
labelMapper := bleve.NewDocumentMapping()
mapper.AddSubDocumentMapping(resource.SEARCH_FIELD_LABELS, labelMapper)
@@ -36,6 +36,7 @@ func TestDocumentMapping(t *testing.T) {
TimestampMillis: 1234,
},
}
data.UpdateCopyFields()
doc := document.NewDocument("id")
err = mappings.MapDocument(doc, data)
@@ -47,5 +48,5 @@ func TestDocumentMapping(t *testing.T) {
fmt.Printf("DOC: fields %d\n", len(doc.Fields))
fmt.Printf("DOC: size %d\n", doc.Size())
require.Equal(t, 15, len(doc.Fields))
require.Equal(t, 16, len(doc.Fields))
}
+15 -37
View File
@@ -8,13 +8,14 @@ import (
"testing"
"github.com/blevesearch/bleve/v2"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/stretchr/testify/require"
)
func TestCanSearchByTitle(t *testing.T) {
@@ -35,9 +36,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "bbb",
TitleNgram: "bbb",
TitlePhrase: "bbb",
Title: "bbb",
})
require.NoError(t, err)
err = index.Write(&resource.IndexableDocument{
@@ -49,9 +48,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "aaa",
TitleNgram: "aaa",
TitlePhrase: "aaa",
Title: "aaa",
})
require.NoError(t, err)
@@ -74,9 +71,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "I want to say a hello",
TitleNgram: "I want to say a hello",
TitlePhrase: "I want to say a hello",
Title: "I want to say a hello",
})
require.NoError(t, err)
err = index.Write(&resource.IndexableDocument{
@@ -88,9 +83,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "we want hello",
TitleNgram: "we want hello",
TitlePhrase: "we want hello",
Title: "we want hello",
})
require.NoError(t, err)
@@ -113,9 +106,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "Asserts Dashboards",
TitleNgram: "Asserts Dashboards",
TitlePhrase: "Asserts Dashboards",
Title: "Asserts Dashboards",
})
require.NoError(t, err)
err = index.Write(&resource.IndexableDocument{
@@ -127,9 +118,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "New dashboard 10",
TitleNgram: "New dashboard 10",
TitlePhrase: "New dashboard 10",
Title: "New dashboard 10",
})
require.NoError(t, err)
@@ -151,9 +140,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "we want hello pls",
TitleNgram: "we want hello pls",
TitlePhrase: "we want hello pls",
Title: "we want hello pls",
})
require.NoError(t, err)
err = index.Write(&resource.IndexableDocument{
@@ -165,9 +152,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "we want hello",
TitleNgram: "we want hello",
TitlePhrase: "we want hello",
Title: "we want hello",
})
require.NoError(t, err)
@@ -190,8 +175,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "A123456",
TitleNgram: "A123456",
Title: "A123456",
})
require.NoError(t, err)
@@ -219,9 +203,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "I want to say a wonderfully Hello to the WORLD! Hello-world",
TitleNgram: "I want to say a wonderfully Hello to the WORLD! Hello-world",
TitlePhrase: "I want to say a wonderfully Hello to the WORLD! Hello-world",
Title: "I want to say a wonderfully Hello to the WORLD! Hello-world",
})
require.NoError(t, err)
@@ -279,8 +261,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "I want to say a wonderful Hello to the WORLD! Hello-world",
TitleNgram: "I want to say a wonderful Hello to the WORLD! Hello-world",
Title: "I want to say a wonderful Hello to the WORLD! Hello-world",
})
require.NoError(t, err)
err = index.Write(&resource.IndexableDocument{
@@ -292,8 +273,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "A0456",
TitleNgram: "A0456",
Title: "A0456",
})
require.NoError(t, err)
err = index.Write(&resource.IndexableDocument{
@@ -305,9 +285,7 @@ func TestCanSearchByTitle(t *testing.T) {
Group: key.Group,
Resource: key.Resource,
},
Title: "mash-A02382-10",
TitleNgram: "mash-A02382-10",
TitlePhrase: "mash-A02382-10",
Title: "mash-A02382-10",
})
require.NoError(t, err)
+17 -19
View File
@@ -76,9 +76,8 @@ func TestBleveBackend(t *testing.T) {
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
Title: "aaa (dash)",
TitlePhrase: "aaa (dash)",
Folder: "xxx",
Title: "aaa (dash)",
Folder: "xxx",
Fields: map[string]any{
DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"},
DASHBOARD_ERRORS_TODAY: 25,
@@ -107,9 +106,8 @@ func TestBleveBackend(t *testing.T) {
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
Title: "bbb (dash)",
TitlePhrase: "bbb (dash)",
Folder: "xxx",
Title: "bbb (dash)",
Folder: "xxx",
Fields: map[string]any{
DASHBOARD_PANEL_TYPES: []string{"timeseries"},
DASHBOARD_ERRORS_TODAY: 40,
@@ -138,10 +136,9 @@ func TestBleveBackend(t *testing.T) {
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
Name: "ccc",
Title: "ccc (dash)",
TitlePhrase: "ccc (dash)",
Folder: "zzz",
Name: "ccc",
Title: "ccc (dash)",
Folder: "zzz",
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo2",
@@ -263,8 +260,9 @@ func TestBleveBackend(t *testing.T) {
require.Equal(t, 0, len(rsp.Results.Rows))
// Now look for repositories
found, err := index.ListRepositoryObjects(ctx, &resource.ListRepositoryObjectsRequest{
Name: "repo-1",
found, err := index.ListManagedObjects(ctx, &resource.ListManagedObjectsRequest{
Kind: "repo",
Id: "repo-1",
})
require.NoError(t, err)
jj, err := json.MarshalIndent(found, "", " ")
@@ -302,20 +300,22 @@ func TestBleveBackend(t *testing.T) {
]
}`, string(jj))
counts, err := index.CountRepositoryObjects(ctx)
counts, err := index.CountManagedObjects(ctx)
require.NoError(t, err)
jj, err = json.MarshalIndent(counts, "", " ")
require.NoError(t, err)
fmt.Printf("%s\n", string(jj))
require.JSONEq(t, `[
{
"repository": "repo-1",
"kind": "repo",
"id": "repo-1",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"count": 2
},
{
"repository": "repo2",
"kind": "repo",
"id": "repo2",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"count": 1
@@ -340,8 +340,7 @@ func TestBleveBackend(t *testing.T) {
Group: "folder.grafana.app",
Resource: "folders",
},
Title: "zzz (folder)",
TitlePhrase: "zzz (folder)",
Title: "zzz (folder)",
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
@@ -360,8 +359,7 @@ func TestBleveBackend(t *testing.T) {
Group: "folder.grafana.app",
Resource: "folders",
},
Title: "yyy (folder)",
TitlePhrase: "yyy (folder)",
Title: "yyy (folder)",
Labels: map[string]string{
"region": "west",
},
@@ -15,5 +15,6 @@
"manager": {
"kind": "repo",
"id": "MyGIT"
}
},
"managedBy": "repo:MyGIT"
}
@@ -15,5 +15,6 @@
"manager": {
"kind": "repo",
"id": "MyGIT"
}
},
"managedBy": "repo:MyGIT"
}
+2 -2
View File
@@ -53,11 +53,11 @@ func (b *backend) GetStats(ctx context.Context, req *resource.ResourceStatsReque
return rsp, nil
}
func (b *backend) RepositoryList(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
func (b *backend) RepositoryList(ctx context.Context, req *resource.ListManagedObjectsRequest) (*resource.ListManagedObjectsResponse, error) {
return nil, fmt.Errorf("SQL backend does not implement RepositoryList")
}
func (b *backend) RepositoryStats(context.Context, *resource.CountRepositoryObjectsRequest) (*resource.CountRepositoryObjectsResponse, error) {
func (b *backend) RepositoryStats(context.Context, *resource.CountManagedObjectsRequest) (*resource.CountManagedObjectsResponse, error) {
return nil, fmt.Errorf("SQL backend does not implement RepositoryStats")
}
+1 -1
View File
@@ -133,7 +133,7 @@ func (s *service) start(ctx context.Context) error {
resource.RegisterResourceStoreServer(srv, server)
resource.RegisterBulkStoreServer(srv, server)
resource.RegisterResourceIndexServer(srv, server)
resource.RegisterRepositoryIndexServer(srv, server)
resource.RegisterManagedObjectIndexServer(srv, server)
resource.RegisterBlobStoreServer(srv, server)
resource.RegisterDiagnosticsServer(srv, server)
grpc_health_v1.RegisterHealthServer(srv, healthService)