public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wire up the ui to the new search api (#97866)

Browse Source 
wire up the ui to the new search api

Co-authored-by: Scott Lepper <scott.lepper@gmail.com>
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Dan Cech <dcech@grafana.com>
This commit is contained in:
Scott Lepper
2024-12-19 13:21:48 -05:00
committed by GitHub
co-authored by Ryan McKinley Dan Cech
parent 86a13ea4bd
commit a8f347144d
27 changed files with 563 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-data/src/types/featureToggles.gen.ts
+1
View File
@@ -243,6 +243,7 @@ export interface FeatureToggles {
alertingNotificationsStepMode?: boolean;
useV2DashboardsAPI?: boolean;
feedbackButton?: boolean;
unifiedStorageSearchUI?: boolean;
elasticsearchCrossClusterSearch?: boolean;
unifiedHistory?: boolean;
lokiLabelNamesQueryApi?: boolean;
pkg/apis/dashboard/v0alpha1/search.go
+2 -12
View File
@@ -51,26 +51,16 @@ type SortableField struct {
Type string `json:"type,omitempty"` // string or number
}
// Dashboard or folder hit
// +enum
type HitKind string
// PluginType values
const (
HitTypeDash HitKind = "Dashboard"
HitTypeFolder HitKind = "Folder"
)
type DashboardHit struct {
// Dashboard or folder
Kind HitKind `json:"kind"`
Resource string `json:"resource"` // dashboards | folders
// The k8s "name" (eg, grafana UID)
Name string `json:"name"`
// The display nam
Title string `json:"title"`
// Filter tags
Tags []string `json:"tags,omitempty"`
// The UID/name for the folder
// The k8s name (eg, grafana UID) for the parent folder
Folder string `json:"folder,omitempty"`
// Stick untyped extra fields in this object (including the sort value)
Field *common.Unstructured `json:"field,omitempty"`
pkg/apis/dashboard/v0alpha1/zz_generated.openapi.go
+4 -5
View File
@@ -216,13 +216,12 @@ func schema_pkg_apis_dashboard_v0alpha1_DashboardHit(ref common.ReferenceCallbac
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"kind": {
"resource": {
SchemaProps: spec.SchemaProps{
Description: "Dashboard or folder\n\nPossible enum values:\n - `\"Dashboard\"`\n - `\"Folder\"`",
Description: "Dashboard or folder",
Default: "",
Type: []string{"string"},
Format: "",
Enum: []interface{}{"Dashboard", "Folder"},
},
},
"name": {
@@ -258,7 +257,7 @@ func schema_pkg_apis_dashboard_v0alpha1_DashboardHit(ref common.ReferenceCallbac
},
"folder": {
SchemaProps: spec.SchemaProps{
Description: "The UID/name for the folder",
Description: "The k8s name (eg, grafana UID) for the parent folder",
Type: []string{"string"},
Format: "",
},
@@ -283,7 +282,7 @@ func schema_pkg_apis_dashboard_v0alpha1_DashboardHit(ref common.ReferenceCallbac
},
},
},
Required: []string{"kind", "name", "title"},
Required: []string{"resource", "name", "title"},
},
},
Dependencies: []string{
pkg/registry/apis/dashboard/search.go
+77 -23
View File
@@ -8,6 +8,7 @@ import (
"strings"
"go.opentelemetry.io/otel/trace"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
@@ -217,16 +218,10 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) {
}
searchRequest := &resource.ResourceSearchRequest{
Options: &resource.ListOptions{
Key: &resource.ResourceKey{
Namespace: user.GetNamespace(),
Group: dashboardv0alpha1.GROUP,
Resource: "dashboards",
},
},
Query: queryParams.Get("query"),
Limit: int64(limit),
Offset: int64(offset),
Options: &resource.ListOptions{},
Query: queryParams.Get("query"),
Limit: int64(limit),
Offset: int64(offset),
Fields: []string{
"title",
"folder",
@@ -244,6 +239,34 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) {
}}
}
types := queryParams["type"]
var federate *resource.ResourceKey
switch len(types) {
case 0:
// When no type specified, search for dashboards
searchRequest.Options.Key, err = asResourceKey(user.GetNamespace(), "dashboards")
// Currently a search query is across folders and dashboards
if searchRequest.Query != "" {
federate, err = asResourceKey(user.GetNamespace(), "folders")
}
case 1:
searchRequest.Options.Key, err = asResourceKey(user.GetNamespace(), types[0])
case 2:
searchRequest.Options.Key, err = asResourceKey(user.GetNamespace(), types[0])
if err == nil {
federate, err = asResourceKey(user.GetNamespace(), types[1])
}
default:
err = apierrors.NewBadRequest("too many type requests")
}
if err != nil {
errhttp.Write(ctx, err, w)
return
}
if federate != nil {
searchRequest.Federated = []*resource.ResourceKey{federate}
}
// Add sorting
if queryParams.Has("sort") {
for _, sort := range queryParams["sort"] {
@@ -256,15 +279,6 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) {
}
}
// Also query folders
if searchRequest.Query != "" {
searchRequest.Federated = []*resource.ResourceKey{{
Namespace: searchRequest.Options.Key.Namespace,
Group: "folder.grafana.app",
Resource: "folders",
}}
}
// The facet term fields
facets, ok := queryParams["facet"]
if ok {
@@ -277,6 +291,16 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) {
}
}
// The tags filter
tags, ok := queryParams["tag"]
if ok {
searchRequest.Options.Fields = []*resource.Requirement{{
Key: "tags",
Operator: "=",
Values: tags,
}}
}
// Run the query
result, err := s.client.Search(ctx, searchRequest)
if err != nil {
@@ -293,10 +317,10 @@ func (s *SearchHandler) DoSearch(w http.ResponseWriter, r *http.Request) {
}
for i, row := range result.Results.Rows {
hit := &dashboardv0alpha1.DashboardHit{
Kind: dashboardv0alpha1.HitTypeDash,
Name: row.Key.Name,
Title: string(row.Cells[0]),
Folder: string(row.Cells[1]),
Resource: row.Key.Resource, // folders | dashboards
Name: row.Key.Name, // The Grafana UID
Title: string(row.Cells[0]),
Folder: string(row.Cells[1]),
}
if row.Cells[2] != nil {
_ = json.Unmarshal(row.Cells[2], &hit.Tags)
@@ -330,3 +354,33 @@ func (s *SearchHandler) write(w http.ResponseWriter, obj any) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(obj)
}
// Given a namespace and type convert it to a search key
func asResourceKey(ns string, k string) (*resource.ResourceKey, error) {
if ns == "" {
return nil, apierrors.NewBadRequest("missing namespace")
}
switch k {
case "folders", "folder":
return &resource.ResourceKey{
Namespace: ns,
Group: "folder.grafana.app",
Resource: "folders",
}, nil
case "dashboards", "dashboard":
return &resource.ResourceKey{
Namespace: ns,
Group: dashboardv0alpha1.GROUP,
Resource: "dashboards",
}, nil
// NOT really supported in the dashboard search UI, but useful for manual testing
case "playlist", "playlists":
return &resource.ResourceKey{
Namespace: ns,
Group: "playlist.grafana.app",
Resource: "playlists",
}, nil
}
return nil, apierrors.NewBadRequest("unknown resource type")
}
pkg/services/featuremgmt/registry.go
+8
View File
@@ -1684,6 +1684,14 @@ var (
Owner: grafanaOperatorExperienceSquad,
HideFromDocs: true,
},
{
Name: "unifiedStorageSearchUI",
Description: "Enable unified storage search UI",
Stage: FeatureStageExperimental,
Owner: grafanaSearchAndStorageSquad,
HideFromDocs: true,
HideFromAdminPage: true,
},
{
Name: "elasticsearchCrossClusterSearch",
Description: "Enables cross cluster search in the Elasticsearch datasource",
pkg/services/featuremgmt/toggles_gen.csv
+1
View File
@@ -224,6 +224,7 @@ azureMonitorEnableUserAuth,GA,@grafana/partner-datasources,false,false,false
alertingNotificationsStepMode,experimental,@grafana/alerting-squad,false,false,true
useV2DashboardsAPI,experimental,@grafana/dashboards-squad,false,true,false
feedbackButton,experimental,@grafana/grafana-operator-experience-squad,false,false,false
unifiedStorageSearchUI,experimental,@grafana/search-and-storage,false,false,false
elasticsearchCrossClusterSearch,preview,@grafana/aws-datasources,false,false,false
unifiedHistory,experimental,@grafana/grafana-frontend-platform,false,false,true
lokiLabelNamesQueryApi,GA,@grafana/observability-logs,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
224 alertingNotificationsStepMode experimental @grafana/alerting-squad false false true
225 useV2DashboardsAPI experimental @grafana/dashboards-squad false true false
226 feedbackButton experimental @grafana/grafana-operator-experience-squad false false false
227 unifiedStorageSearchUI experimental @grafana/search-and-storage false false false
228 elasticsearchCrossClusterSearch preview @grafana/aws-datasources false false false
229 unifiedHistory experimental @grafana/grafana-frontend-platform false false true
230 lokiLabelNamesQueryApi GA @grafana/observability-logs false false false
pkg/services/featuremgmt/toggles_gen.go
+4
View File
@@ -907,6 +907,10 @@ const (
// Enables a button to send feedback from the Grafana UI
FlagFeedbackButton = "feedbackButton"
// FlagUnifiedStorageSearchUI
// Enable unified storage search UI
FlagUnifiedStorageSearchUI = "unifiedStorageSearchUI"
// FlagElasticsearchCrossClusterSearch
// Enables cross cluster search in the Elasticsearch datasource
FlagElasticsearchCrossClusterSearch = "elasticsearchCrossClusterSearch"
pkg/services/featuremgmt/toggles_gen.json
+19 -2
View File
@@ -3635,8 +3635,8 @@
{
"metadata": {
"name": "unifiedStorageSearchSprinkles",
"resourceVersion": "1734123247356",
"creationTimestamp": "2024-12-13T20:54:07Z"
"resourceVersion": "1734563607668",
"creationTimestamp": "2024-12-18T23:13:27Z"
},
"spec": {
"description": "Enable sprinkles on unified storage search",
@@ -3646,6 +3646,23 @@
"hideFromDocs": true
}
},
{
"metadata": {
"name": "unifiedStorageSearchUI",
"resourceVersion": "1734563607668",
"creationTimestamp": "2024-12-10T21:28:55Z",
"annotations": {
"grafana.app/updatedTimestamp": "2024-12-18 23:13:27.66802 +0000 UTC"
}
},
"spec": {
"description": "Enable unified storage search UI",
"stage": "experimental",
"codeowner": "@grafana/search-and-storage",
"hideFromAdminPage": true,
"hideFromDocs": true
}
},
{
"metadata": {
"name": "useSeessionStorageForRedirection",
pkg/storage/unified/resource/document.go
+29 -2
View File
@@ -53,12 +53,18 @@ type IndexableDocument struct {
// The resource key
Key *ResourceKey `json:"key"`
// The resource type ( for federated indexes )
Kind string `json:"kind,omitempty"`
// Resource version for the resource (if known)
RV int64 `json:"rv,omitempty"`
// The generic display name
Title string `json:"title,omitempty"`
// internal sort field for title ( don't set this directly )
TitleSort string `json:"title_sort,omitempty"`
// A generic description -- helpful in global search
Description string `json:"description,omitempty"`
@@ -99,6 +105,10 @@ type IndexableDocument struct {
RepoInfo *utils.ResourceRepositoryInfo `json:"repository,omitempty"`
}
func (m *IndexableDocument) Type() string {
return m.Key.Resource
}
type ResourceReference struct {
Relation string `json:"relation"` // eg: depends-on
Group string `json:"group,omitempty"` // the api group
@@ -138,10 +148,26 @@ func (m ResourceReferences) Less(i, j int) bool {
// Create a new indexable document based on a generic k8s resource
func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAccessor) *IndexableDocument {
title := obj.FindTitle(key.Name)
if title == key.Name {
// TODO: something wrong with FindTitle
spec, err := obj.GetSpec()
if err == nil {
specValue, ok := spec.(map[string]any)
if ok {
specTitle, ok := specValue["title"].(string)
if ok {
title = specTitle
}
}
}
}
doc := &IndexableDocument{
Key: key,
Kind: key.Resource,
RV: rv,
Title: obj.FindTitle(key.Name), // We always want *something* to display
Title: title, // We always want *something* to display
TitleSort: title,
Labels: obj.GetLabels(),
Folder: obj.GetFolder(),
CreatedBy: obj.GetCreatedBy(),
@@ -178,7 +204,6 @@ func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *Resour
}
doc := NewIndexableDocument(key, rv, obj)
doc.Title = obj.FindTitle(key.Name)
return doc, nil
}
@@ -220,11 +245,13 @@ func (x *searchableDocumentFields) Field(name string) *ResourceTableColumnDefini
}
const SEARCH_FIELD_ID = "_id" // {namespace}/{group}/{resource}/{name}
const SEARCH_FIELD_KIND = "kind" // resource ( for federated index filtering )
const SEARCH_FIELD_GROUP_RESOURCE = "gr" // group/resource
const SEARCH_FIELD_NAMESPACE = "namespace"
const SEARCH_FIELD_NAME = "name"
const SEARCH_FIELD_RV = "rv"
const SEARCH_FIELD_TITLE = "title"
const SEARCH_FIELD_TITLE_SORT = "title_sort"
const SEARCH_FIELD_DESCRIPTION = "description"
const SEARCH_FIELD_TAGS = "tags"
const SEARCH_FIELD_LABELS = "labels" // All labels, not a specific one
pkg/storage/unified/resource/document_test.go
+4 -2
View File
@@ -33,8 +33,10 @@ func TestStandardDocumentBuilder(t *testing.T) {
"resource": "playlists",
"name": "test1"
},
"kind": "playlists",
"rv": 10,
"title": "test1",
"title": "test playlist unified storage",
"title_sort": "test playlist unified storage",
"created": 1717236672000,
"createdBy": "user:ABC",
"updatedBy": "user:XYZ",
@@ -43,5 +45,5 @@ func TestStandardDocumentBuilder(t *testing.T) {
"path": "15",
"hash": "xyz"
}
}`, string(jj))
}`, string(jj))
}
pkg/storage/unified/search/bleve.go
+91 -53
View File
@@ -305,27 +305,9 @@ func (b *bleveIndex) Search(
return nil, err
}
// Write frame as JSON
//response.Frame, err = frame.MarshalJSON()
if err != nil {
return nil, err
}
// parse the facet fields
for k, v := range res.Facets {
f := &resource.ResourceSearchResponse_Facet{
Field: v.Field,
Total: int64(v.Total),
Missing: int64(v.Missing),
}
if v.Terms != nil {
for _, t := range v.Terms.Terms() {
f.Terms = append(f.Terms, &resource.ResourceSearchResponse_TermFacet{
Term: t.Term,
Count: int64(t.Count),
})
}
}
f := newResponseFacet(v)
if response.Facet == nil {
response.Facet = make(map[string]*resource.ResourceSearchResponse_Facet)
}
@@ -387,7 +369,7 @@ func (b *bleveIndex) getIndex(
return nil, fmt.Errorf("federated indexes must be the same type")
}
if typedindex.verifyKey(req.Federated[i]) != nil {
return nil, fmt.Errorf("federated index keys do not match")
return nil, fmt.Errorf("federated index keys do not match (%v != %v)", typedindex, req.Federated[i])
}
all = append(all, typedindex.index)
}
@@ -397,11 +379,16 @@ func (b *bleveIndex) getIndex(
}
func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.AccessClient) (*bleve.SearchRequest, *resource.ErrorResult) {
facets := bleve.FacetsRequest{}
for _, f := range req.Facet {
facets[f.Field] = bleve.NewFacetRequest(f.Field, int(f.Limit))
}
searchrequest := &bleve.SearchRequest{
Fields: req.Fields,
Size: int(req.Limit),
From: int(req.Offset),
Explain: req.Explain,
Facets: facets,
}
// Currently everything is within an AND query
@@ -415,6 +402,7 @@ func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.Acce
queries = append(queries, q)
}
}
// filters
if len(req.Options.Fields) > 0 {
for _, v := range req.Options.Fields {
q, err := requirementQuery(v, "")
@@ -425,12 +413,7 @@ func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.Acce
}
}
if req.Query != "" {
// ??? Should expose the full power of query parsing here?
// it is great for exploration, but also hard to change in the future
q := bleve.NewQueryStringQuery(req.Query)
queries = append(queries, q)
}
queries = append(queries, newTextQuery(req))
if access != nil {
// TODO AUTHZ!!!!
@@ -458,30 +441,11 @@ func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.Acce
}
// Add the sort fields
for _, sort := range req.SortBy {
// hardcoded (for now)
if strings.HasPrefix(sort.Field, "stats.") {
searchrequest.Sort = append(searchrequest.Sort, &search.SortField{
Field: sort.Field,
Desc: sort.Desc,
Type: search.SortFieldAsNumber, // force for now!
Mode: search.SortFieldDefault, // ???
Missing: search.SortFieldMissingLast,
})
continue
}
// Default support
input := sort.Field
if sort.Desc {
input = "-" + sort.Field
}
s := search.ParseSearchSortString(input)
searchrequest.Sort = append(searchrequest.Sort, s)
}
sorting := getSortFields(req)
searchrequest.SortBy(sorting)
// Always sort by *something*, otherwise the order is unstable
if len(searchrequest.Sort) == 0 {
if len(sorting) == 0 {
searchrequest.Sort = append(searchrequest.Sort, &search.SortDocID{
Desc: false,
})
@@ -490,23 +454,71 @@ func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.Acce
return searchrequest, nil
}
func getSortFields(req *resource.ResourceSearchRequest) []string {
sorting := []string{}
for _, sort := range req.SortBy {
input := sort.Field
if field, ok := textSortFields[input]; ok {
input = field
}
if sort.Desc {
input = "-" + input
}
sorting = append(sorting, input)
}
return sorting
}
// fields that we went to sort by the full text
var textSortFields = map[string]string{
resource.SEARCH_FIELD_TITLE: resource.SEARCH_FIELD_TITLE + "_sort",
}
// Convert a "requirement" into a bleve query
func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *resource.ErrorResult) {
switch selection.Operator(req.Operator) {
case selection.Equals, selection.DoubleEquals:
if len(req.Values) != 1 {
return nil, resource.NewBadRequestError("equals query can have one value")
if len(req.Values) == 0 {
return query.NewMatchAllQuery(), nil
}
if len(req.Values[0]) == 1 {
q := query.NewMatchQuery(req.Values[0])
q.FieldVal = prefix + req.Key
return q, nil
}
q := query.NewMatchQuery(req.Values[0])
q.FieldVal = prefix + req.Key
return q, nil
conjuncts := []query.Query{}
for _, v := range req.Values {
q := query.NewMatchQuery(v)
q.FieldVal = prefix + req.Key
conjuncts = append(conjuncts, q)
}
return query.NewConjunctionQuery(conjuncts), nil
case selection.NotEquals:
case selection.DoesNotExist:
case selection.GreaterThan:
case selection.LessThan:
case selection.Exists:
case selection.In:
if len(req.Values) == 0 {
return query.NewMatchAllQuery(), nil
}
if len(req.Values) == 1 {
q := query.NewMatchQuery(req.Values[0])
q.FieldVal = prefix + req.Key
return q, nil
}
disjuncts := []query.Query{}
for _, v := range req.Values {
q := query.NewMatchQuery(v)
q.FieldVal = prefix + req.Key
disjuncts = append(disjuncts, q)
}
return query.NewDisjunctionQuery(disjuncts), nil
case selection.NotIn:
}
return nil, resource.NewBadRequestError(
@@ -615,3 +627,29 @@ func getAllFields(standard resource.SearchableDocumentFields, custom resource.Se
}
return fields, nil
}
func newResponseFacet(v *search.FacetResult) *resource.ResourceSearchResponse_Facet {
f := &resource.ResourceSearchResponse_Facet{
Field: v.Field,
Total: int64(v.Total),
Missing: int64(v.Missing),
}
if v.Terms != nil {
for _, t := range v.Terms.Terms() {
f.Terms = append(f.Terms, &resource.ResourceSearchResponse_TermFacet{
Term: t.Term,
Count: int64(t.Count),
})
}
}
return f
}
func newTextQuery(req *resource.ResourceSearchRequest) query.Query {
if req.Query == "" || req.Query == "*" {
return bleve.NewMatchAllQuery()
}
// TODO: wildcard query?
// return bleve.NewWildcardQuery(req.Query)
return bleve.NewFuzzyQuery(req.Query)
}
pkg/storage/unified/search/bleve_mappings.go
+25 -21
View File
@@ -16,21 +16,20 @@ func getBleveMappings(fields resource.SearchableDocumentFields) mapping.IndexMap
func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentMapping {
mapper := bleve.NewDocumentStaticMapping()
mapper.AddFieldMapping(&mapping.FieldMapping{
Name: resource.SEARCH_FIELD_TITLE,
Type: "text",
// TODO - if we don't want title to be a keyword, we can use this
// set the title field to use keyword analyzer so it sorts by the whole phrase
// https://github.com/blevesearch/bleve/issues/417#issuecomment-245273022
Analyzer: keyword.Name,
Store: true,
Index: true,
IncludeTermVectors: true,
IncludeInAll: true,
DocValues: false,
})
mapper.AddFieldMapping(&mapping.FieldMapping{
// for sorting by title
titleSortMapping := bleve.NewKeywordFieldMapping()
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE_SORT, titleSortMapping)
// for searching by title
titleSearchMapping := bleve.NewTextFieldMapping()
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TITLE, titleSearchMapping)
// for filtering by kind/resource ( federated search )
kindMapping := bleve.NewTextFieldMapping()
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_KIND, kindMapping)
descriptionMapping := &mapping.FieldMapping{
Name: resource.SEARCH_FIELD_DESCRIPTION,
Type: "text",
Store: true,
@@ -38,9 +37,10 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: false,
DocValues: false,
})
}
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_DESCRIPTION, descriptionMapping)
mapper.AddFieldMapping(&mapping.FieldMapping{
tagsMapping := &mapping.FieldMapping{
Name: resource.SEARCH_FIELD_TAGS,
Type: "text",
Analyzer: keyword.Name,
@@ -49,9 +49,10 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
DocValues: false,
})
}
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_TAGS, tagsMapping)
mapper.AddFieldMapping(&mapping.FieldMapping{
folderMapping := &mapping.FieldMapping{
Name: resource.SEARCH_FIELD_FOLDER,
Type: "text",
Analyzer: keyword.Name,
@@ -60,9 +61,10 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
DocValues: true, // will be needed for authz client
})
}
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_FOLDER, folderMapping)
mapper.AddFieldMapping(&mapping.FieldMapping{
repoMapping := &mapping.FieldMapping{
Name: resource.SEARCH_FIELD_REPOSITORY,
Type: "text",
Analyzer: keyword.Name,
@@ -71,8 +73,10 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
DocValues: true,
})
}
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_REPOSITORY, repoMapping)
// TODO: we use the static mapper. why set dynamic to true?
mapper.Dynamic = true
return mapper
pkg/storage/unified/search/bleve_mappings_test.go
+1 -1
View File
@@ -42,5 +42,5 @@ func TestDocumentMapping(t *testing.T) {
fmt.Printf("DOC: fields %d\n", len(doc.Fields))
fmt.Printf("DOC: size %d\n", doc.Size())
require.Equal(t, 15, len(doc.Fields))
require.Equal(t, 17, len(doc.Fields))
}
pkg/storage/unified/search/bleve_test.go
+24 -25
View File
@@ -66,11 +66,12 @@ func TestBleveBackend(t *testing.T) {
Key: &resource.ResourceKey{
Name: "aaa",
Namespace: "ns",
Group: "g",
Resource: "dash",
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
Title: "aaa (dash)",
Folder: "xxx",
Title: "aaa (dash)",
TitleSort: "aaa (dash)",
Folder: "xxx",
Fields: map[string]any{
DASHBOARD_LEGACY_ID: 12,
DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"},
@@ -83,11 +84,12 @@ func TestBleveBackend(t *testing.T) {
Key: &resource.ResourceKey{
Name: "bbb",
Namespace: "ns",
Group: "g",
Resource: "dash",
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
Title: "bbb (dash)",
Folder: "xxx",
Title: "bbb (dash)",
TitleSort: "bbb (dash)",
Folder: "xxx",
Fields: map[string]any{
DASHBOARD_LEGACY_ID: 12,
DASHBOARD_PANEL_TYPES: []string{"timeseries"},
@@ -103,11 +105,12 @@ func TestBleveBackend(t *testing.T) {
Key: &resource.ResourceKey{
Name: "ccc",
Namespace: "ns",
Group: "g",
Resource: "dash",
Group: "dashboard.grafana.app",
Resource: "dashboards",
},
Title: "ccc (dash)",
Folder: "zzz",
Title: "ccc (dash)",
TitleSort: "ccc (dash)",
Folder: "zzz",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "r0",
},
@@ -145,11 +148,7 @@ func TestBleveBackend(t *testing.T) {
require.NotNil(t, rsp.Results)
require.NotNil(t, rsp.Facet)
t.Run("x", func(t *testing.T) {
t.Skip("flakey tests - skipping") // sort seems different in CI... sometimes!
// Match the results
resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-dashboard.json"), rsp.Results)
})
resource.AssertTableSnapshot(t, filepath.Join("testdata", "manual-dashboard.json"), rsp.Results)
// Get the tags facets
facet, ok := rsp.Facet["tags"]
@@ -193,20 +192,22 @@ func TestBleveBackend(t *testing.T) {
Key: &resource.ResourceKey{
Name: "zzz",
Namespace: "ns",
Group: "g",
Resource: "folder",
Group: "folder.grafana.app",
Resource: "folders",
},
Title: "zzz (folder)",
Title: "zzz (folder)",
TitleSort: "zzz (folder)",
})
_ = index.Write(&resource.IndexableDocument{
RV: 2,
Key: &resource.ResourceKey{
Name: "yyy",
Namespace: "ns",
Group: "g",
Resource: "folder",
Group: "folder.grafana.app",
Resource: "folders",
},
Title: "yyy (folder)",
Title: "yyy (folder)",
TitleSort: "yyy (folder)",
Labels: map[string]string{
"region": "west",
},
@@ -232,8 +233,6 @@ func TestBleveBackend(t *testing.T) {
})
t.Run("simple federation", func(t *testing.T) {
t.Skip("flakey tests - skipping") // sort seems different in CI... sometimes!
// The other tests must run first to build the indexes
require.NotNil(t, dashboardsIndex)
require.NotNil(t, foldersIndex)
pkg/storage/unified/search/testdata/doc/dashboard-aaa-out.json
Vendored
+2
View File
@@ -5,8 +5,10 @@
"resource": "dashboards",
"name": "aaa"
},
"kind": "dashboards",
"rv": 1234,
"title": "Test title",
"title_sort": "Test title",
"description": "test description",
"tags": [
"a",
pkg/storage/unified/search/testdata/doc/folder-aaa-out.json
Vendored
+3 -1
View File
@@ -5,8 +5,10 @@
"resource": "dashboards",
"name": "aaa"
},
"kind": "dashboards",
"rv": 1234,
"title": "aaa",
"title": "test-aaa",
"title_sort": "test-aaa",
"created": 1730490142000,
"createdBy": "user:1",
"repository": {
pkg/storage/unified/search/testdata/doc/folder-bbb-out.json
Vendored
+3 -1
View File
@@ -5,8 +5,10 @@
"resource": "dashboards",
"name": "bbb"
},
"kind": "dashboards",
"rv": 1234,
"title": "bbb",
"title": "test-bbb",
"title_sort": "test-bbb",
"created": 1730490142000,
"createdBy": "user:1",
"repository": {
pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json
Vendored
+3 -1
View File
@@ -5,8 +5,10 @@
"resource": "dashboards",
"name": "aaa"
},
"kind": "dashboards",
"rv": 1234,
"title": "aaa",
"title": "Test AAA",
"title_sort": "Test AAA",
"created": 1731336353000,
"createdBy": "user:t000000001",
"repository": {
pkg/storage/unified/search/testdata/doc/report-aaa-out.json
Vendored
+3 -1
View File
@@ -5,8 +5,10 @@
"resource": "dashboards",
"name": "aaa"
},
"kind": "dashboards",
"rv": 1234,
"title": "aaa",
"title": "Test AAA",
"title_sort": "Test AAA",
"created": 1706690655000,
"createdBy": "user:abc",
"repository": {
pkg/storage/unified/search/testdata/folder-federated.json
Vendored
+51
View File
@@ -0,0 +1,51 @@
{
"metadata": {},
"columnDefinitions": [
{
"name": "title",
"type": "string",
"format": "",
"description": "Display name for the resource",
"priority": 0
},
{
"name": "_id",
"type": "string",
"format": "",
"description": "Unique Identifier. {namespace}/{group}/{resource}/{name}",
"priority": 0
}
],
"rows": [
{
"cells": [
"yyy (folder)",
"ns/folder.grafana.app/folders/yyy"
],
"object": {
"kind": "folders",
"apiVersion": "folder.grafana.app",
"metadata": {
"name": "yyy",
"namespace": "ns",
"creationTimestamp": null
}
}
},
{
"cells": [
"zzz (folder)",
"ns/folder.grafana.app/folders/zzz"
],
"object": {
"kind": "folders",
"apiVersion": "folder.grafana.app",
"metadata": {
"name": "zzz",
"namespace": "ns",
"creationTimestamp": null
}
}
}
]
}
pkg/storage/unified/search/testdata/manual-dashboard.json
Vendored
+9 -9
View File
@@ -68,7 +68,7 @@
"rows": [
{
"cells": [
"ns/g/dash/ccc",
"ns/dashboard.grafana.app/dashboards/ccc",
"ccc (dash)",
[
"aa"
@@ -81,8 +81,8 @@
null
],
"object": {
"kind": "dash",
"apiVersion": "g",
"kind": "dashboards",
"apiVersion": "dashboard.grafana.app",
"metadata": {
"name": "ccc",
"namespace": "ns",
@@ -92,7 +92,7 @@
},
{
"cells": [
"ns/g/dash/bbb",
"ns/dashboard.grafana.app/dashboards/bbb",
"bbb (dash)",
[
"aa"
@@ -105,8 +105,8 @@
null
],
"object": {
"kind": "dash",
"apiVersion": "g",
"kind": "dashboards",
"apiVersion": "dashboard.grafana.app",
"metadata": {
"name": "bbb",
"namespace": "ns",
@@ -116,7 +116,7 @@
},
{
"cells": [
"ns/g/dash/aaa",
"ns/dashboard.grafana.app/dashboards/aaa",
"aaa (dash)",
[
"aa",
@@ -130,8 +130,8 @@
null
],
"object": {
"kind": "dash",
"apiVersion": "g",
"kind": "dashboards",
"apiVersion": "dashboard.grafana.app",
"metadata": {
"name": "aaa",
"namespace": "ns",
pkg/storage/unified/search/testdata/manual-federated.json
Vendored
+28 -28
View File
@@ -20,26 +20,11 @@
{
"cells": [
"aaa (dash)",
"ns/g/dash/bbb"
"ns/dashboard.grafana.app/dashboards/aaa"
],
"object": {
"kind": "dash",
"apiVersion": "g",
"metadata": {
"name": "bbb",
"namespace": "ns",
"creationTimestamp": null
}
}
},
{
"cells": [
"bbb (dash)",
"ns/g/dash/aaa"
],
"object": {
"kind": "dash",
"apiVersion": "g",
"kind": "dashboards",
"apiVersion": "dashboard.grafana.app",
"metadata": {
"name": "aaa",
"namespace": "ns",
@@ -49,12 +34,27 @@
},
{
"cells": [
"ccc (dash)",
"ns/g/dash/ccc"
"bbb (dash)",
"ns/dashboard.grafana.app/dashboards/bbb"
],
"object": {
"kind": "dash",
"apiVersion": "g",
"kind": "dashboards",
"apiVersion": "dashboard.grafana.app",
"metadata": {
"name": "bbb",
"namespace": "ns",
"creationTimestamp": null
}
}
},
{
"cells": [
"ccc (dash)",
"ns/dashboard.grafana.app/dashboards/ccc"
],
"object": {
"kind": "dashboards",
"apiVersion": "dashboard.grafana.app",
"metadata": {
"name": "ccc",
"namespace": "ns",
@@ -65,11 +65,11 @@
{
"cells": [
"yyy (folder)",
"ns/g/folder/yyy"
"ns/folder.grafana.app/folders/yyy"
],
"object": {
"kind": "folder",
"apiVersion": "g",
"kind": "folders",
"apiVersion": "folder.grafana.app",
"metadata": {
"name": "yyy",
"namespace": "ns",
@@ -80,11 +80,11 @@
{
"cells": [
"zzz (folder)",
"ns/g/folder/zzz"
"ns/folder.grafana.app/folders/zzz"
],
"object": {
"kind": "folder",
"apiVersion": "g",
"kind": "folders",
"apiVersion": "folder.grafana.app",
"metadata": {
"name": "zzz",
"namespace": "ns",
pkg/storage/unified/search/testdata/manual-folder.json
Vendored
+6 -6
View File
@@ -47,7 +47,7 @@
"rows": [
{
"cells": [
"ns/g/folder/yyy",
"ns/folder.grafana.app/folders/yyy",
"yyy (folder)",
null,
null,
@@ -55,8 +55,8 @@
0
],
"object": {
"kind": "folder",
"apiVersion": "g",
"kind": "folders",
"apiVersion": "folder.grafana.app",
"metadata": {
"name": "yyy",
"namespace": "ns",
@@ -66,7 +66,7 @@
},
{
"cells": [
"ns/g/folder/zzz",
"ns/folder.grafana.app/folders/zzz",
"zzz (folder)",
null,
null,
@@ -74,8 +74,8 @@
0
],
"object": {
"kind": "folder",
"apiVersion": "g",
"kind": "folders",
"apiVersion": "folder.grafana.app",
"metadata": {
"name": "zzz",
"namespace": "ns",
pkg/util/errhttp/writer.go
+11
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"reflect"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apiserver/pkg/endpoints/request"
"github.com/grafana/grafana/pkg/apimachinery/errutil"
@@ -40,6 +41,16 @@ func Write(ctx context.Context, err error, w http.ResponseWriter, opts ...func(E
var gErr errutil.Error
if !errors.As(err, &gErr) {
// Write k8s response if this is a k8s error
k8s, ok := err.(apierrors.APIStatus)
if ok {
status := k8s.Status()
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(int(status.Code))
_ = json.NewEncoder(w).Encode(status)
return
}
gErr = fallbackOrInternalError(err, opt)
}
public/app/features/browse-dashboards/api/services.ts
+1
View File
@@ -23,6 +23,7 @@ export async function listFolders(
const backendSrv = getBackendSrv();
// TODO: what to do here for unified search?
let folders: NestedFolderDTO[] = [];
if (contextSrv.hasPermission(AccessControlAction.FoldersRead)) {
folders = await backendSrv.get<NestedFolderDTO[]>('/api/folders', {
public/app/features/search/service/searcher.ts
+1 -1
View File
@@ -18,7 +18,7 @@ export function getGrafanaSearcher(): GrafanaSearcher {
return new FrontendSearcher(searcher);
}
const useUnifiedStorageSearch = false; // TODO, frontend FF config.featureToggles.unifiedStorageSearch;
const useUnifiedStorageSearch = config.featureToggles.unifiedStorageSearchUI;
searcher = useUnifiedStorageSearch ? new UnifiedSearcher(sqlSearcher) : sqlSearcher;
}
return searcher!;
public/app/features/search/service/unified.ts
+152 -77
View File
@@ -1,36 +1,68 @@
import {
DataFrame,
DataFrameJSON,
DataFrameView,
getDisplayProcessor,
SelectableValue,
toDataFrame,
} from '@grafana/data';
import { isEmpty } from 'lodash';
import { DataFrame, DataFrameView, getDisplayProcessor, SelectableValue, toDataFrame } from '@grafana/data';
import { config, getBackendSrv } from '@grafana/runtime';
import { TermCount } from 'app/core/components/TagFilter/TagFilter';
import { DashboardQueryResult, GrafanaSearcher, QueryResponse, SearchQuery, SearchResultMeta } from './types';
import {
DashboardQueryResult,
GrafanaSearcher,
LocationInfo,
QueryResponse,
SearchQuery,
SearchResultMeta,
} from './types';
import { replaceCurrentFolderQuery } from './utils';
// The backend returns an empty frame with a special name to indicate that the indexing engine is being rebuilt,
// and that it can not serve any search requests. We are temporarily using the old SQL Search API as a fallback when that happens.
const loadingFrameName = 'Loading';
const searchURI = 'api/unified-search';
const searchURI = `apis/dashboard.grafana.app/v0alpha1/namespaces/${config.namespace}/search`;
type SearchHit = {
resource: string; // dashboards | folders
name: string;
title: string;
location: string;
folder: string;
tags: string[];
// calculated in the frontend
url: string;
};
type SearchAPIResponse = {
frames: DataFrameJSON[];
hits: SearchHit[];
facets?: {
tags?: {
terms?: Array<{
term: string;
count: number;
}>;
};
};
};
const folderViewSort = 'name_sort';
export class UnifiedSearcher implements GrafanaSearcher {
constructor(private fallbackSearcher: GrafanaSearcher) {}
locationInfo: Promise<Record<string, LocationInfo>>;
constructor(private fallbackSearcher: GrafanaSearcher) {
this.locationInfo = loadLocationInfo();
}
async search(query: SearchQuery): Promise<QueryResponse> {
if (query.facet?.length) {
throw new Error('facets not supported!');
}
if (query.kind?.length === 1 && query.kind[0] === 'dashboard') {
// TODO: this is browse mode, so skip the search
return noDataResponse();
}
return this.doSearchQuery(query);
}
@@ -47,41 +79,14 @@ export class UnifiedSearcher implements GrafanaSearcher {
});
}
// Nothing is starred
return {
view: new DataFrameView({ length: 0, fields: [] }),
totalRows: 0,
loadMoreItems: async (startIndex: number, stopIndex: number): Promise<void> => {
return;
},
isItemLoaded: (index: number): boolean => {
return true;
},
};
return noDataResponse();
}
async tags(query: SearchQuery): Promise<TermCount[]> {
const req = {
...query,
query: query.query ?? '*',
sort: undefined, // no need to sort the initial query results (not used)
facet: [{ field: 'tags' }],
limit: 1, // 0 would be better, but is ignored by the backend
};
const resp = await getBackendSrv().post<SearchAPIResponse>(searchURI, req);
const frames = resp.frames.map((f) => toDataFrame(f));
if (frames[0]?.name === loadingFrameName) {
return this.fallbackSearcher.tags(query);
}
for (const frame of frames) {
if (frame.name === 'tags') {
return getTermCountsFrom(frame);
}
}
return [];
const qry = query.query ?? '*';
let uri = `${searchURI}?facet=tags&query=${qry}&limit=1`;
const resp = await getBackendSrv().get<SearchAPIResponse>(uri);
return resp.facets?.tags?.terms || [];
}
// TODO: Implement this correctly
@@ -113,34 +118,37 @@ export class UnifiedSearcher implements GrafanaSearcher {
limit: query.limit ?? firstPageSize,
};
const rsp = await getBackendSrv().post<SearchAPIResponse>(searchURI, req);
const frames = rsp.frames.map((f) => toDataFrame(f));
let uri = searchURI;
const qry = req.query || '*';
uri += `?query=${encodeURIComponent(qry)}`;
if (req.limit) {
uri += `&limit=${req.limit}`;
}
const first = frames.length ? toDataFrame(frames[0]) : { fields: [], length: 0 };
if (req.kind) {
// filter resource types
uri += '&' + req.kind.map((kind) => `type=${kind}`).join('&');
}
if (req.tags) {
uri += '&' + req.tags.map((tag) => `tag=${encodeURIComponent(tag)}`).join('&');
}
const rsp = await getBackendSrv().get<SearchAPIResponse>(uri);
const first = toDashboardResults(rsp.hits);
if (first.name === loadingFrameName) {
return this.fallbackSearcher.search(query);
}
for (const field of first.fields) {
field.display = getDisplayProcessor({ field, theme: config.theme2 });
}
// Make sure the object exists
if (!first.meta?.custom) {
first.meta = {
...first.meta,
custom: {
count: first.length,
max_score: 1,
},
};
}
const meta = first.meta.custom as SearchResultMeta;
if (!meta.locationInfo) {
meta.locationInfo = {}; // always set it so we can append
const meta = first.meta?.custom || ({} as SearchResultMeta);
const locationInfo = await this.locationInfo;
const hasMissing = rsp.hits.some((hit) => !locationInfo[hit.folder]);
if (hasMissing) {
// sync the location info ( folders )
this.locationInfo = loadLocationInfo();
}
meta.locationInfo = await this.locationInfo;
// Set the field name to a better display name
if (meta.sortBy?.length) {
@@ -155,6 +163,7 @@ export class UnifiedSearcher implements GrafanaSearcher {
let loadMax = 0;
let pending: Promise<void> | undefined = undefined;
const getNextPage = async () => {
// TODO: implement this correctly
while (loadMax > view.dataFrame.length) {
const from = view.dataFrame.length;
if (from >= meta.count) {
@@ -165,8 +174,7 @@ export class UnifiedSearcher implements GrafanaSearcher {
from,
limit: nextPageSizes,
});
const frame = toDataFrame(resp.frames[0]);
const frame = toDashboardResults(resp.hits);
if (!frame) {
console.log('no results', frame);
return;
@@ -220,16 +228,6 @@ export class UnifiedSearcher implements GrafanaSearcher {
const firstPageSize = 50;
const nextPageSizes = 100;
function getTermCountsFrom(frame: DataFrame): TermCount[] {
const tags = frame.fields[0].values;
const vals = frame.fields[1].values;
const counts: TermCount[] = [];
for (let i = 0; i < frame.length; i++) {
counts.push({ term: tags[i], count: vals[i] });
}
return counts;
}
// Enterprise only sort field values for dashboards
const sortFields = [
{ name: 'views_total', display: 'Views total' },
@@ -244,6 +242,19 @@ const sortTimeFields = [
{ name: 'updated_at', display: 'Updated time' },
];
function noDataResponse(): QueryResponse | PromiseLike<QueryResponse> {
return {
view: new DataFrameView({ length: 0, fields: [] }),
totalRows: 0,
loadMoreItems: async (startIndex: number, stopIndex: number): Promise<void> => {
return;
},
isItemLoaded: (index: number): boolean => {
return true;
},
};
}
/** Given the internal field name, this gives a reasonable display name for the table colum header */
function getSortFieldDisplayName(name: string) {
for (const sf of sortFields) {
@@ -258,3 +269,67 @@ function getSortFieldDisplayName(name: string) {
}
return name;
}
function toDashboardResults(hits: SearchHit[]): DataFrame {
if (hits.length < 1) {
return { fields: [], length: 0 };
}
const dashboardHits = hits.map((hit) => {
let location = hit.folder;
if (hit.resource === 'dashboards' && isEmpty(location)) {
location = 'general';
}
return {
...hit,
url: toURL(hit.resource, hit.name),
tags: hit.tags || [],
folder: hit.folder || 'general',
location,
name: hit.title, // 🤯 FIXME hit.name is k8s name, eg grafana dashboards UID
kind: hit.resource.substring(0, hit.resource.length - 1), // dashboard "kind" is not plural
};
});
const frame = toDataFrame(dashboardHits);
frame.meta = {
custom: {
count: hits.length,
max_score: 1,
},
};
for (const field of frame.fields) {
field.display = getDisplayProcessor({ field, theme: config.theme2 });
}
return frame;
}
async function loadLocationInfo(): Promise<Record<string, LocationInfo>> {
const uri = `${searchURI}?type=folders`;
const rsp = getBackendSrv()
.get<SearchAPIResponse>(uri)
.then((rsp) => {
const locationInfo: Record<string, LocationInfo> = {
general: {
kind: 'folder',
name: 'Dashboards',
url: '/dashboards',
}, // share location info with everyone
};
for (const hit of rsp.hits) {
locationInfo[hit.name] = {
name: hit.title,
kind: 'folder',
url: toURL('folders', hit.name),
};
}
return locationInfo;
});
return rsp;
}
function toURL(resource: string, name: string): string {
if (resource === 'folders') {
return `/dashboards/f/${name}`;
}
return `/d/${name}`;
}