diff --git a/apps/folder/kinds/folder.cue b/apps/folder/kinds/folder.cue index 24ba0c83dd4..496e34281fc 100644 --- a/apps/folder/kinds/folder.cue +++ b/apps/folder/kinds/folder.cue @@ -8,6 +8,12 @@ foldersV1beta1: { spec: { title: string description?: string + foo: bool + bar: int } } -} \ No newline at end of file + + selectableFields: [ + "spec.title", + ] +} diff --git a/apps/folder/pkg/apis/folder/v1beta1/folder_schema_gen.go b/apps/folder/pkg/apis/folder/v1beta1/folder_schema_gen.go index e626e4773ee..7044290fda6 100644 --- a/apps/folder/pkg/apis/folder/v1beta1/folder_schema_gen.go +++ b/apps/folder/pkg/apis/folder/v1beta1/folder_schema_gen.go @@ -5,13 +5,26 @@ package v1beta1 import ( + "errors" + "github.com/grafana/grafana-app-sdk/resource" ) // schema is unexported to prevent accidental overwrites var ( schemaFolder = resource.NewSimpleSchema("folder.grafana.app", "v1beta1", NewFolder(), &FolderList{}, resource.WithKind("Folder"), - resource.WithPlural("folders"), resource.WithScope(resource.NamespacedScope)) + resource.WithPlural("folders"), resource.WithScope(resource.NamespacedScope), resource.WithSelectableFields([]resource.SelectableField{resource.SelectableField{ + FieldSelector: "spec.title", + FieldValueFunc: func(o resource.Object) (string, error) { + cast, ok := o.(*Folder) + if !ok { + return "", errors.New("provided object must be of type *Folder") + } + + return cast.Spec.Title, nil + }, + }, + })) kindFolder = resource.Kind{ Schema: schemaFolder, Codecs: map[resource.KindEncoding]resource.Codec{ diff --git a/apps/folder/pkg/apis/manifestdata/folder_manifest.go b/apps/folder/pkg/apis/manifestdata/folder_manifest.go index 7c053e52c38..a66903ca44c 100644 --- a/apps/folder/pkg/apis/manifestdata/folder_manifest.go +++ b/apps/folder/pkg/apis/manifestdata/folder_manifest.go @@ -18,6 +18,8 @@ import ( v1beta1 "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" ) +var () + var appManifestData = app.ManifestData{ AppName: "folder", Group: "folder.grafana.app", @@ -32,6 +34,9 @@ var appManifestData = app.ManifestData{ Plural: "Folders", Scope: "Namespaced", Conversion: false, + SelectableFields: []string{ + "spec.title", + }, }, }, Routes: app.ManifestVersionRoutes{ diff --git a/apps/iam/kinds/user.cue b/apps/iam/kinds/user.cue index a67c0949e1f..a1b1b0271ef 100644 --- a/apps/iam/kinds/user.cue +++ b/apps/iam/kinds/user.cue @@ -14,7 +14,7 @@ userKind: { } userv0alpha1: userKind & { - // TODO: Uncomment this when User will be added to ManagedKinds + // TODO: Uncomment this when User will be added to ManagedKinds // validation: { // operations: [ // "CREATE", diff --git a/apps/iam/pkg/apis/iam_manifest.go b/apps/iam/pkg/apis/iam_manifest.go index 0106984f7f2..3b7a0751fa5 100644 --- a/apps/iam/pkg/apis/iam_manifest.go +++ b/apps/iam/pkg/apis/iam_manifest.go @@ -74,6 +74,10 @@ var appManifestData = app.ManifestData{ Plural: "Users", Scope: "Namespaced", Conversion: false, + SelectableFields: []string{ + "spec.email", + "spec.login", + }, }, { diff --git a/pkg/apiserver/registry/generic/storage.go b/pkg/apiserver/registry/generic/storage.go index 98e2f1fe9df..440fed45398 100644 --- a/pkg/apiserver/registry/generic/storage.go +++ b/pkg/apiserver/registry/generic/storage.go @@ -4,11 +4,24 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/generic/registry" + "k8s.io/apiserver/pkg/storage" "github.com/grafana/grafana/pkg/apimachinery/utils" ) -func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter) (*registry.Store, error) { +type registryStoreOptions struct { + attrFunc storage.AttrFunc +} + +type OptionFn func(*registryStoreOptions) + +func WithAttrFunc(attrFunc storage.AttrFunc) OptionFn { + return func(opts *registryStoreOptions) { + opts.attrFunc = attrFunc + } +} + +func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter, options ...OptionFn) (*registry.Store, error) { gv := resourceInfo.GroupVersion() gv.Version = runtime.APIVersionInternal strategy := NewStrategy(scheme, gv) @@ -20,7 +33,7 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o NewListFunc: resourceInfo.NewListFunc, KeyRootFunc: KeyRootFunc(resourceInfo.GroupResource()), KeyFunc: NamespaceKeyFunc(resourceInfo.GroupResource()), - PredicateFunc: Matcher, + //PredicateFunc: Matcher, DefaultQualifiedResource: resourceInfo.GroupResource(), SingularQualifiedResource: resourceInfo.SingularGroupResource(), TableConvertor: resourceInfo.TableConverter(), @@ -28,8 +41,16 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o UpdateStrategy: strategy, DeleteStrategy: strategy, } - options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs} - if err := store.CompleteWithOptions(options); err != nil { + + opts := ®istryStoreOptions{ + attrFunc: GetAttrs, + } + for _, opt := range options { + opt(opts) + } + + o := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: opts.attrFunc} + if err := store.CompleteWithOptions(o); err != nil { return nil, err } return store, nil diff --git a/pkg/registry/apis/folders/register.go b/pkg/registry/apis/folders/register.go index 64e51c06312..e1e916f77a0 100644 --- a/pkg/registry/apis/folders/register.go +++ b/pkg/registry/apis/folders/register.go @@ -8,6 +8,8 @@ import ( "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" @@ -22,6 +24,8 @@ import ( authlib "github.com/grafana/authlib/types" "github.com/grafana/grafana-app-sdk/logging" + sdkres "github.com/grafana/grafana-app-sdk/resource" + folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" "github.com/grafana/grafana/apps/iam/pkg/reconcilers" "github.com/grafana/grafana/pkg/apimachinery/utils" @@ -129,6 +133,29 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { Version: runtime.APIVersionInternal, }) + kinds := []sdkres.Kind{folders.FolderKind()} + for _, kind := range kinds { + gvk := gv.WithKind(kind.Kind()) + err := scheme.AddFieldLabelConversionFunc( + gvk, + func(label, value string) (string, string, error) { + if label == "metadata.name" || label == "metadata.namespace" { + return label, value, nil + } + fields := kind.SelectableFields() + for _, field := range fields { + if field.FieldSelector == label { + return label, value, nil + } + } + return "", "", fmt.Errorf("field label not supported for %s: %s", gvk, label) + }, + ) + if err != nil { + return err + } + } + // If multiple versions exist, then register conversions from zz_generated.conversion.go // if err := playlist.RegisterConversions(scheme); err != nil { // return err @@ -137,6 +164,26 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(gv) } +// TODO: work with all kinds from schema, not just one. +func (b *FolderAPIBuilder) BuildGetAttrsFn(k sdkres.Kind) func(obj runtime.Object) (labels.Set, fields.Set, error) { + return func(obj runtime.Object) (labels.Set, fields.Set, error) { + if robj, ok := obj.(sdkres.Object); !ok { + return nil, nil, fmt.Errorf("not a resource.Object") + } else { + fieldsSet := fields.Set{} + + for _, f := range k.SelectableFields() { + v, err := f.FieldValueFunc(robj) + if err != nil { + return nil, nil, err + } + fieldsSet[f.FieldSelector] = v + } + return robj.GetLabels(), fieldsSet, nil + } + } +} + func (b *FolderAPIBuilder) AllowedV0Alpha1Resources() []string { return nil } @@ -148,10 +195,11 @@ func (b *FolderAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.API Permissions: b.setDefaultFolderPermissions, }) - unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter) + unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter, grafanaregistry.WithAttrFunc(b.BuildGetAttrsFn(folders.FolderKind()))) if err != nil { return err } + b.registerPermissionHooks(unified) b.storage = unified diff --git a/pkg/storage/unified/resource/document.go b/pkg/storage/unified/resource/document.go index 4e528b96df0..6949e11823e 100644 --- a/pkg/storage/unified/resource/document.go +++ b/pkg/storage/unified/resource/document.go @@ -3,9 +3,11 @@ package resource import ( "context" "fmt" + "strconv" "strings" "sync" + "github.com/grafana/grafana-app-sdk/app" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -101,6 +103,9 @@ type IndexableDocument struct { // metadata, annotations, or external data linked at index time Fields map[string]any `json:"fields,omitempty"` + // Automatically indexed selectable fields, used for field-based filtering when listing. + SelectableFields map[string]string `json:"selectable_fields,omitempty"` + // Maintain a list of resource references. // Someday this will likely be part of https://github.com/grafana/gamma References ResourceReferences `json:"references,omitempty"` @@ -175,7 +180,7 @@ func (m ResourceReferences) Less(i, j int) bool { } // Create a new indexable document based on a generic k8s resource -func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.GrafanaMetaAccessor) *IndexableDocument { +func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.GrafanaMetaAccessor, selectableFields map[string]string) *IndexableDocument { title := obj.FindTitle(key.Name) if title == key.Name { // TODO: something wrong with FindTitle @@ -191,14 +196,15 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa } } doc := &IndexableDocument{ - Key: key, - RV: rv, - Name: key.Name, - Title: title, // We always want *something* to display - Labels: obj.GetLabels(), - Folder: obj.GetFolder(), - CreatedBy: obj.GetCreatedBy(), - UpdatedBy: obj.GetUpdatedBy(), + Key: key, + RV: rv, + Name: key.Name, + Title: title, // We always want *something* to display + Labels: obj.GetLabels(), + Folder: obj.GetFolder(), + CreatedBy: obj.GetCreatedBy(), + UpdatedBy: obj.GetUpdatedBy(), + SelectableFields: selectableFields, } m, ok := obj.GetManagerProperties() if ok { @@ -220,11 +226,14 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa return doc.UpdateCopyFields() } -func StandardDocumentBuilder() DocumentBuilder { - return &standardDocumentBuilder{} +func StandardDocumentBuilder(manifests []app.Manifest) DocumentBuilder { + return &standardDocumentBuilder{selectableFields: SelectableFieldsForManifests(manifests)} } -type standardDocumentBuilder struct{} +type standardDocumentBuilder struct { + // Maps "group/resource" (in lowercase) to list of selectable fields. + selectableFields map[string][]string +} func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*IndexableDocument, error) { tmp := &unstructured.Unstructured{} @@ -238,10 +247,36 @@ func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resour return nil, err } - doc := NewIndexableDocument(key, rv, obj) + sfKey := strings.ToLower(key.GetGroup() + "/" + key.GetResource()) + selectableFields := buildSelectableFields(tmp, s.selectableFields[sfKey]) + + doc := NewIndexableDocument(key, rv, obj, selectableFields) return doc, nil } +func buildSelectableFields(tmp *unstructured.Unstructured, fields []string) map[string]string { + result := map[string]string{} + + for _, field := range fields { + path := strings.Split(field, ".") + val, ok, err := unstructured.NestedFieldNoCopy(tmp.Object, path...) + if err != nil || !ok { + continue + } + + switch v := val.(type) { + case string: + result[field] = v + case bool: + result[field] = strconv.FormatBool(v) + case int, float64: + result[field] = fmt.Sprintf("%v", v) + } + } + + return result +} + type searchableDocumentFields struct { names []string fields map[string]*resourceTableColumn diff --git a/pkg/storage/unified/resource/document_test.go b/pkg/storage/unified/resource/document_test.go index f6bf17531d8..6cd9e67a0bf 100644 --- a/pkg/storage/unified/resource/document_test.go +++ b/pkg/storage/unified/resource/document_test.go @@ -14,7 +14,7 @@ import ( func TestStandardDocumentBuilder(t *testing.T) { ctx := context.Background() - builder := StandardDocumentBuilder() + builder := StandardDocumentBuilder(nil) body, err := os.ReadFile("testdata/playlist-resource.json") require.NoError(t, err) diff --git a/pkg/storage/unified/resource/selectable_fields.go b/pkg/storage/unified/resource/selectable_fields.go new file mode 100644 index 00000000000..03e64972a7f --- /dev/null +++ b/pkg/storage/unified/resource/selectable_fields.go @@ -0,0 +1,45 @@ +package resource + +import ( + "strings" + + "github.com/grafana/grafana-app-sdk/app" + + folder "github.com/grafana/grafana/apps/folder/pkg/apis/manifestdata" + iam "github.com/grafana/grafana/apps/iam/pkg/apis" +) + +func AppManifests() []app.Manifest { + return []app.Manifest{ + iam.LocalManifest(), + folder.LocalManifest(), + } +} + +func SelectableFields() map[string][]string { + return SelectableFieldsForManifests(AppManifests()) +} + +// SelectableFieldsForManifests returns map of to list of selectable fields. +// Also is included as a key, pointing to the same fields. +func SelectableFieldsForManifests(manifests []app.Manifest) map[string][]string { + fields := map[string][]string{} + + for _, m := range manifests { + group := m.ManifestData.Group + + for _, version := range m.ManifestData.Versions { + for _, kind := range version.Kinds { + key := strings.ToLower(group + "/" + kind.Kind) + keyPlural := strings.ToLower(group + "/" + kind.Plural) + + if len(kind.SelectableFields) > 0 { + fields[key] = kind.SelectableFields + fields[keyPlural] = kind.SelectableFields + } + } + } + } + + return fields +} diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index 7c890e72b00..2356826229e 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -12,6 +12,8 @@ import ( "github.com/Masterminds/semver" "github.com/google/uuid" + claims "github.com/grafana/authlib/types" + "github.com/grafana/dskit/backoff" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -20,9 +22,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - claims "github.com/grafana/authlib/types" - "github.com/grafana/dskit/backoff" - "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/apimachinery/validation" "github.com/grafana/grafana/pkg/infra/log" @@ -1043,6 +1042,68 @@ func (s *server) List(ctx context.Context, req *resourcepb.ListRequest) (*resour return rsp, nil } + // Remove metadata.namespace filter from requirement fields, if it's present. + for ix := 0; ix < len(req.Options.Fields); { + v := req.Options.Fields[ix] + if v.Key == "metadata.namespace" && v.Operator == "=" { + if len(v.Values) == 1 && v.Values[0] == req.Options.Key.Namespace { + // Remove this requirement from fields, as it's implied by the key.namespace. + req.Options.Fields = append(req.Options.Fields[:ix], req.Options.Fields[ix+1:]...) + // Don't increment ix, as we're removing an element from the slice. + continue + } + } + ix++ + } + + // TODO: What to do about RV and version_match fields? + // If we get here, we're doing list with selectable fields. Let's do search instead, since + // we index all selectable fields, and fetch resulting documents one by one. + if s.search != nil && req.Source == resourcepb.ListRequest_STORE && (len(req.Options.Fields) > 0) { + if req.Options.Key.Namespace == "" { + return &resourcepb.ListResponse{ + Error: NewBadRequestError("namespace must be specified for list with filter"), + }, nil + } + + srq := &resourcepb.ResourceSearchRequest{ + Options: req.Options, + //Federated: nil, + Limit: req.Limit, + // Offset: req.NextPageToken, // TODO + // Page: 0, + // Permission: 0, // Not needed, default is List + } + + searchResp, err := s.search.Search(ctx, srq) + if err != nil { + return nil, err + } + + rsp := &resourcepb.ListResponse{} + // Using searchResp.GetResults().GetRows() will not panic if anything is nil on the path. + for _, row := range searchResp.GetResults().GetRows() { + // TODO: use batch reading + val, err := s.Read(ctx, &resourcepb.ReadRequest{ + Key: row.Key, + ResourceVersion: row.ResourceVersion, + }) + if err != nil { + return &resourcepb.ListResponse{Error: AsErrorResult(err)}, nil + } + if len(val.Value) > 0 { + rsp.Items = append(rsp.Items, &resourcepb.ResourceWrapper{ + Value: val.Value, + ResourceVersion: val.ResourceVersion, + }) + if val.ResourceVersion > rsp.ResourceVersion { + rsp.ResourceVersion = val.ResourceVersion + } + } + } + return rsp, nil + } + if req.Limit < 1 { req.Limit = 500 // default max 500 items in a page } diff --git a/pkg/storage/unified/resource/storage_backend.go b/pkg/storage/unified/resource/storage_backend.go index 0de97b0355e..a15840e28fd 100644 --- a/pkg/storage/unified/resource/storage_backend.go +++ b/pkg/storage/unified/resource/storage_backend.go @@ -59,7 +59,6 @@ type kvStorageBackend struct { dataStore *dataStore eventStore *eventStore notifier *notifier - builder DocumentBuilder log logging.Logger withPruner bool eventRetentionPeriod time.Duration @@ -109,8 +108,7 @@ func NewKVStorageBackend(opts KVBackendOptions) (StorageBackend, error) { eventStore: eventStore, notifier: newNotifier(eventStore, notifierOptions{}), snowflake: s, - builder: StandardDocumentBuilder(), // For now we use the standard document builder. - log: &logging.NoOpLogger{}, // Make this configurable + log: &logging.NoOpLogger{}, // Make this configurable eventRetentionPeriod: eventRetentionPeriod, eventPruningInterval: eventPruningInterval, withExperimentalClusterScope: opts.WithExperimentalClusterScope, diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index eb9fa4df3bd..0aa75f40105 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -93,6 +93,8 @@ type bleveBackend struct { indexMetrics *resource.BleveIndexMetrics + selectableFields map[string][]string + bgTasksCancel func() bgTasksWg sync.WaitGroup } @@ -135,11 +137,12 @@ func NewBleveBackend(opts BleveOptions, indexMetrics *resource.BleveIndexMetrics } be := &bleveBackend{ - log: l, - cache: map[resource.NamespacedResource]*bleveIndex{}, - opts: opts, - ownsIndexFn: ownFn, - indexMetrics: indexMetrics, + log: l, + cache: map[resource.NamespacedResource]*bleveIndex{}, + opts: opts, + ownsIndexFn: ownFn, + indexMetrics: indexMetrics, + selectableFields: resource.SelectableFields(), } ctx, cancel := context.WithCancel(context.Background()) @@ -366,7 +369,9 @@ func (b *bleveBackend) BuildIndex( attribute.String("reason", indexBuildReason), ) - mapper, err := GetBleveMappings(fields) + selectableFields := b.selectableFields[fmt.Sprintf("%s/%s", key.Group, key.Resource)] + + mapper, err := GetBleveMappings(fields, selectableFields) if err != nil { return nil, err } @@ -459,7 +464,7 @@ func (b *bleveBackend) BuildIndex( } // Batch all the changes - idx := b.newBleveIndex(key, index, newIndexType, fields, allFields, standardSearchFields, updater, b.log.New("namespace", key.Namespace, "group", key.Group, "resource", key.Resource)) + idx := b.newBleveIndex(key, index, newIndexType, fields, allFields, standardSearchFields, selectableFields, updater, b.log.New("namespace", key.Namespace, "group", key.Group, "resource", key.Resource)) if build { if b.indexMetrics != nil { @@ -699,8 +704,9 @@ type bleveIndex struct { // Subsequent update requests only trigger new update if minUpdateInterval has elapsed. nextUpdateTime time.Time - standard resource.SearchableDocumentFields - fields resource.SearchableDocumentFields + standard resource.SearchableDocumentFields + fields resource.SearchableDocumentFields + selectableFields []string indexStorage string // memory or file, used when updating metrics @@ -736,6 +742,7 @@ func (b *bleveBackend) newBleveIndex( fields resource.SearchableDocumentFields, allFields []*resourcepb.ResourceTableColumnDefinition, standardSearchFields resource.SearchableDocumentFields, + selectableFields []string, updaterFn resource.UpdateFn, logger log.Logger, ) *bleveIndex { @@ -745,6 +752,7 @@ func (b *bleveBackend) newBleveIndex( indexStorage: newIndexType, fields: fields, allFields: allFields, + selectableFields: selectableFields, standard: standardSearchFields, logger: logger, updaterFn: updaterFn, @@ -1215,7 +1223,11 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R // filters if len(req.Options.Fields) > 0 { for _, v := range req.Options.Fields { - q, err := requirementQuery(v, "") + prefix := "" + if b.isSelectableField(v.Key) { + prefix = "selectable_fields." + } + q, err := requirementQuery(v, prefix) if err != nil { return nil, err } @@ -1787,6 +1799,15 @@ func (b *bleveIndex) hitsToTable(ctx context.Context, selectFields []string, hit return table, nil } +func (b *bleveIndex) isSelectableField(key string) bool { + for _, f := range b.selectableFields { + if key == f { + return true + } + } + return false +} + func getAllFields(standard resource.SearchableDocumentFields, custom resource.SearchableDocumentFields) ([]*resourcepb.ResourceTableColumnDefinition, error) { fields := []*resourcepb.ResourceTableColumnDefinition{ standard.Field(resource.SEARCH_FIELD_ID), diff --git a/pkg/storage/unified/search/bleve_mappings.go b/pkg/storage/unified/search/bleve_mappings.go index 43adcbc607e..e0dd85ee4fa 100644 --- a/pkg/storage/unified/search/bleve_mappings.go +++ b/pkg/storage/unified/search/bleve_mappings.go @@ -10,19 +10,19 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resourcepb" ) -func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) { +func GetBleveMappings(fields resource.SearchableDocumentFields, selectableFields []string) (mapping.IndexMapping, error) { mapper := bleve.NewIndexMapping() err := RegisterCustomAnalyzers(mapper) if err != nil { return nil, err } - mapper.DefaultMapping = getBleveDocMappings(fields) + mapper.DefaultMapping = getBleveDocMappings(fields, selectableFields) return mapper, nil } -func getBleveDocMappings(fields resource.SearchableDocumentFields) *mapping.DocumentMapping { +func getBleveDocMappings(fields resource.SearchableDocumentFields, selectableFields []string) *mapping.DocumentMapping { mapper := bleve.NewDocumentStaticMapping() nameMapping := &mapping.FieldMapping{ @@ -165,5 +165,73 @@ func getBleveDocMappings(fields resource.SearchableDocumentFields) *mapping.Docu mapper.AddSubDocumentMapping("fields", fieldMapper) + selectableFieldsMapper := bleve.NewDocumentStaticMapping() + for _, field := range selectableFields { + selectableFieldsMapper.AddFieldMappingsAt(field, &mapping.FieldMapping{ + Name: field, + Type: "text", + Analyzer: keyword.Name, + Store: false, + Index: true, + }) + } + mapper.AddSubDocumentMapping("selectable_fields", selectableFieldsMapper) + return mapper } + +/* +Here's a tree representation of the field mappings in pkg/storage/unified/search/bleve_mappings.go: + + Document Root (DefaultMapping) + │ + ├── name [text, keyword analyzer] + │ + ├── title_phrase [keyword, not stored] + │ + ├── title [3 mappings] + │ ├── [1] standard analyzer, stored + │ ├── [2] TITLE_ANALYZER (edge ngram), not stored + │ └── [3] keyword, not stored + │ + ├── description [text, stored] + │ + ├── tags [text, keyword analyzer, stored, includeInAll] + │ + ├── folder [text, keyword analyzer, stored, includeInAll, docValues] + │ + ├── managedBy [text, keyword analyzer, not stored] + │ + ├── source/ [sub-document] + │ ├── path [text, keyword analyzer, stored] + │ ├── checksum [text, keyword analyzer, stored] + │ └── timestampMillis [numeric] + │ + ├── manager/ [sub-document] + │ ├── kind [text, keyword analyzer, stored, includeInAll] + │ └── id [text, keyword analyzer, stored, includeInAll] + │ + ├── reference/ [sub-document, default analyzer: keyword] + │ └── (dynamic fields inherit keyword analyzer) + │ + ├── labels/ [sub-document] + │ └── (dynamic fields) + │ + └── fields/ [sub-document] + └── (conditional mappings) + ├── {filterable string fields} [keyword, stored] + └── {other fields} [dynamically mapped by Bleve] + + Key observations: + + - Root level has standard searchable fields (name, title, description, tags, folder) + - title has 3 analyzers applied: standard (for word search), edge ngram (for prefix search), and keyword (for phrase sorting) + - source/, manager/: Static sub-documents with explicitly mapped fields + - reference/: Dynamic sub-document with keyword default analyzer (line 142) + - labels/, fields/: Dynamic sub-documents where Bleve auto-detects field types at index time + + References: + - Main mapping function: pkg/storage/unified/search/bleve_mappings.go:25-169 + - Sub-document mappings: lines 88-143 + - Dynamic fields handling: lines 148-166 +*/ diff --git a/pkg/storage/unified/search/bleve_mappings_test.go b/pkg/storage/unified/search/bleve_mappings_test.go index 3b8027ee06e..10c0acd4008 100644 --- a/pkg/storage/unified/search/bleve_mappings_test.go +++ b/pkg/storage/unified/search/bleve_mappings_test.go @@ -13,7 +13,7 @@ import ( ) func TestDocumentMapping(t *testing.T) { - mappings, err := search.GetBleveMappings(nil) + mappings, err := search.GetBleveMappings(nil, nil) require.NoError(t, err) data := resource.IndexableDocument{ Title: "title", diff --git a/pkg/storage/unified/search/builders/dashboard.go b/pkg/storage/unified/search/builders/dashboard.go index 4f8d55111a9..d96bcb4ac24 100644 --- a/pkg/storage/unified/search/builders/dashboard.go +++ b/pkg/storage/unified/search/builders/dashboard.go @@ -264,7 +264,7 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou summary.UID = obj.GetName() summary.ID = obj.GetDeprecatedInternalID() // nolint:staticcheck - doc := resource.NewIndexableDocument(key, rv, obj) + doc := resource.NewIndexableDocument(key, rv, obj, nil) doc.Title = summary.Title doc.Description = summary.Description doc.Tags = summary.Tags diff --git a/pkg/storage/unified/search/builders/document_test.go b/pkg/storage/unified/search/builders/document_test.go index 6c8d02d6cb8..1361c92c16f 100644 --- a/pkg/storage/unified/search/builders/document_test.go +++ b/pkg/storage/unified/search/builders/document_test.go @@ -115,7 +115,7 @@ func TestDashboardDocumentBuilder(t *testing.T) { "aaa", }) - builder = resource.StandardDocumentBuilder() + builder = resource.StandardDocumentBuilder(nil) doSnapshotTests(t, builder, "folder", &resourcepb.ResourceKey{ Namespace: "default", Group: "folder.grafana.app", diff --git a/pkg/storage/unified/search/builders/external_group_mapping.go b/pkg/storage/unified/search/builders/external_group_mapping.go index 809075c2ec2..ec12e44908e 100644 --- a/pkg/storage/unified/search/builders/external_group_mapping.go +++ b/pkg/storage/unified/search/builders/external_group_mapping.go @@ -66,7 +66,7 @@ func (u *extGroupMappingDocumentBuilder) BuildDocument(ctx context.Context, key return nil, err } - doc := resource.NewIndexableDocument(key, rv, obj) + doc := resource.NewIndexableDocument(key, rv, obj, nil) doc.Fields = make(map[string]any) if extGroupMapping.Spec.TeamRef.Name != "" { diff --git a/pkg/storage/unified/search/builders/team_search.go b/pkg/storage/unified/search/builders/team_search.go index 4b09074b3dc..b9703d02843 100644 --- a/pkg/storage/unified/search/builders/team_search.go +++ b/pkg/storage/unified/search/builders/team_search.go @@ -70,7 +70,7 @@ func (t *teamSearchBuilder) BuildDocument(ctx context.Context, key *resourcepb.R return nil, err } - doc := resource.NewIndexableDocument(key, rv, obj) + doc := resource.NewIndexableDocument(key, rv, obj, nil) doc.Fields = make(map[string]any) if team.Spec.Email != "" { diff --git a/pkg/storage/unified/search/builders/user.go b/pkg/storage/unified/search/builders/user.go index b17cdb99b15..f74d4fb8c47 100644 --- a/pkg/storage/unified/search/builders/user.go +++ b/pkg/storage/unified/search/builders/user.go @@ -66,7 +66,7 @@ func (u *userDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb return nil, err } - doc := resource.NewIndexableDocument(key, rv, obj) + doc := resource.NewIndexableDocument(key, rv, obj, nil) doc.Fields = make(map[string]any) if user.Spec.Email != "" { diff --git a/pkg/storage/unified/search/document.go b/pkg/storage/unified/search/document.go index fff4c31ead9..206439f3fcf 100644 --- a/pkg/storage/unified/search/document.go +++ b/pkg/storage/unified/search/document.go @@ -25,7 +25,7 @@ func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBui result := []resource.DocumentBuilderInfo{ { - Builder: resource.StandardDocumentBuilder(), + Builder: resource.StandardDocumentBuilder(resource.AppManifests()), }, } return append(result, all...), nil