public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unified Storage: Make all dashboard fields searchable (#98899)

Browse Source 
* wip. adding sprinkles fields.

* some refactoring. Works with sprinkles now.

* exclude top level dashboard hit fields from hit "fields"

* adds unit test for DecodeCell helper

* test can search for specific dashboard fields on bleve index

* adds search handler tests for the fields and tests for fields when transforming the search req to a bleve search req

* fix panic when calling fields.Set() with int32

* adds regression test

* remove unneeded method on test mock client

* fix linter issues

* updates dashboard test data for bleve tests

* remove DASHBOARD_LEGACY_ID from bleve_tests

* dont cast twice

* updates test to sort by dashboard_views_last_1_days

* declare excludedFields outside of function

* fixes sorting by dashboard fields - prepends "fields." to any dashboard fields we try to sort by

* uses map for excludedFields

* expects fields to be array-style url param

* change method name

* fixes failing tests - needed to add column type to mocks
This commit is contained in:
owensmallwood
2025-01-15 10:23:05 -06:00
committed by GitHub
parent 8415089534
commit d00592ffa0
12 changed files with 629 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/resource/document.go
+2
View File
@@ -237,6 +237,8 @@ func (x *searchableDocumentFields) Fields() []string {
}
func (x *searchableDocumentFields) Field(name string) *ResourceTableColumnDefinition {
name = strings.TrimPrefix(name, "fields.")
f, ok := x.fields[name]
if ok {
return f.def
pkg/storage/unified/resource/table.go
+13
View File
@@ -182,6 +182,19 @@ type resourceTableColumn struct {
OpenAPIFormat string
}
// helper to decode a cell value
func DecodeCell(columnDef *ResourceTableColumnDefinition, index int, cellVal []byte) (any, error) {
col, err := newResourceTableColumn(columnDef, index)
if err != nil {
return nil, err
}
res, err := col.Decode(cellVal)
if err != nil {
return nil, err
}
return res, nil
}
// nolint:gocyclo
func newResourceTableColumn(def *ResourceTableColumnDefinition, index int) (*resourceTableColumn, error) {
col := &resourceTableColumn{def: def, index: index}
pkg/storage/unified/resource/table_test.go
+14
View File
@@ -1,6 +1,8 @@
package resource
import (
"bytes"
"encoding/binary"
"fmt"
"path/filepath"
"strings"
@@ -312,3 +314,15 @@ func TestColumnEncoding(t *testing.T) {
require.Empty(t, missingArrays, "missing array tests for types")
})
}
func TestDecodeCell(t *testing.T) {
colDef := &ResourceTableColumnDefinition{Type: ResourceTableColumnDefinition_INT64}
var buf bytes.Buffer
err := binary.Write(&buf, binary.BigEndian, int64(123))
require.NoError(t, err)
res, err := DecodeCell(colDef, 0, buf.Bytes())
require.NoError(t, err)
require.Equal(t, int64(123), res)
}
pkg/storage/unified/search/bleve.go
+20 -2
View File
@@ -7,6 +7,7 @@ import (
"log/slog"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
@@ -489,8 +490,18 @@ func toBleveSearchRequest(req *resource.ResourceSearchRequest, access authz.Acce
for _, f := range req.Facet {
facets[f.Field] = bleve.NewFacetRequest(f.Field, int(f.Limit))
}
// Convert resource-specific fields to bleve fields (just considers dashboard fields for now)
fields := make([]string, 0, len(req.Fields))
for _, f := range req.Fields {
if slices.Contains(DashboardFields(), f) {
f = "fields." + f
}
fields = append(fields, f)
}
searchrequest := &bleve.SearchRequest{
Fields: req.Fields,
Fields: fields,
Size: int(req.Limit),
From: int(req.Offset),
Explain: req.Explain,
@@ -720,7 +731,14 @@ func (b *bleveIndex) hitsToTable(selectFields []string, hits search.DocumentMatc
row.Cells[i], err = json.Marshal(match.Expl)
}
default:
v := match.Fields[f.Name]
fieldName := f.Name
// since the bleve index fields mix common and resource-specific fields, it is possible a conflict can happen
// if a specific field is named the same as a common field
v := match.Fields[fieldName]
// fields that are specific to the resource get stored as fields.<fieldName>, so we need to check for that
if v == nil {
v = match.Fields["fields."+fieldName]
}
if v != nil {
// Encode the value to protobuf
row.Cells[i], err = encoders[i](v)
pkg/storage/unified/search/bleve_mappings.go
+3
View File
@@ -103,5 +103,8 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
labelMapper := bleve.NewDocumentMapping()
mapper.AddSubDocumentMapping(resource.SEARCH_FIELD_LABELS, labelMapper)
fieldMapper := bleve.NewDocumentMapping()
mapper.AddSubDocumentMapping("fields", fieldMapper)
return mapper
}
pkg/storage/unified/search/bleve_test.go
+53 -10
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"testing"
"time"
@@ -76,9 +77,9 @@ func TestBleveBackend(t *testing.T) {
TitleSort: "aaa (dash)",
Folder: "xxx",
Fields: map[string]any{
DASHBOARD_LEGACY_ID: 12,
DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"},
DASHBOARD_ERRORS_TODAY: 25,
DASHBOARD_PANEL_TYPES: []string{"timeseries", "table"},
DASHBOARD_ERRORS_TODAY: 25,
DASHBOARD_VIEWS_LAST_1_DAYS: 50,
},
Labels: map[string]string{
utils.LabelKeyDeprecatedInternalID: "10", // nolint:staticcheck
@@ -104,9 +105,9 @@ func TestBleveBackend(t *testing.T) {
TitleSort: "bbb (dash)",
Folder: "xxx",
Fields: map[string]any{
DASHBOARD_LEGACY_ID: 12,
DASHBOARD_PANEL_TYPES: []string{"timeseries"},
DASHBOARD_ERRORS_TODAY: 40,
DASHBOARD_PANEL_TYPES: []string{"timeseries"},
DASHBOARD_ERRORS_TODAY: 40,
DASHBOARD_VIEWS_LAST_1_DAYS: 100,
},
Tags: []string{"aa"},
Labels: map[string]string{
@@ -136,10 +137,8 @@ func TestBleveBackend(t *testing.T) {
Name: "repo2",
Path: "path/in/repo2.yaml",
},
Fields: map[string]any{
DASHBOARD_LEGACY_ID: 12,
},
Tags: []string{"aa"},
Fields: map[string]any{},
Tags: []string{"aa"},
Labels: map[string]string{
"region": "west",
},
@@ -217,6 +216,27 @@ func TestBleveBackend(t *testing.T) {
rsp.Results.Rows[1].Key.Name,
})
// can get sprinkles fields
rsp, err = index.Search(ctx, nil, &resource.ResourceSearchRequest{
Options: &resource.ListOptions{
Key: key,
},
Limit: 100000,
Fields: []string{DASHBOARD_ERRORS_TODAY, DASHBOARD_VIEWS_LAST_1_DAYS, "fieldThatDoesntExist"},
SortBy: []*resource.ResourceSearchRequest_Sort{
{Field: "fields." + DASHBOARD_VIEWS_LAST_1_DAYS, Desc: true},
},
}, nil)
require.NoError(t, err)
require.Equal(t, 2, len(rsp.Results.Columns))
require.Equal(t, DASHBOARD_ERRORS_TODAY, rsp.Results.Columns[0].Name)
require.Equal(t, DASHBOARD_VIEWS_LAST_1_DAYS, rsp.Results.Columns[1].Name)
// sorted descending so should start with highest dashboard_views_last_1_days (100)
val, err := resource.DecodeCell(rsp.Results.Columns[1], 0, rsp.Results.Rows[0].Cells[1])
require.NoError(t, err)
require.Equal(t, int64(100), val)
// Now look for repositories
found, err := index.ListRepositoryObjects(ctx, &resource.ListRepositoryObjectsRequest{
Name: "repo-1",
@@ -408,6 +428,29 @@ func TestBleveBackend(t *testing.T) {
})
}
func TestToBleveSearchRequest(t *testing.T) {
t.Run("will prepend 'fields.' to all dashboard fields", func(t *testing.T) {
fields := []string{"title", "name", "folder"}
fields = append(fields, DashboardFields()...)
resReq := &resource.ResourceSearchRequest{
Options: &resource.ListOptions{},
Fields: fields,
}
bleveReq, err := toBleveSearchRequest(resReq, nil)
if err != nil {
t.Fatalf("error creating bleve search request: %v", err)
}
require.Equal(t, len(fields), len(bleveReq.Fields))
for _, field := range DashboardFields() {
require.True(t, slices.Contains(bleveReq.Fields, "fields."+field))
}
require.Contains(t, bleveReq.Fields, "title")
require.Contains(t, bleveReq.Fields, "name")
require.Contains(t, bleveReq.Fields, "folder")
})
}
func asTimePointer(milli int64) *time.Time {
if milli > 0 {
t := time.UnixMilli(milli)
pkg/storage/unified/search/dashboard.go
+153
View File
@@ -69,6 +69,126 @@ func DashboardBuilder(namespaced resource.NamespacedDocumentSupplier) (resource.
Filterable: true,
},
},
{
Name: DASHBOARD_ERRORS_TODAY,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of errors that occurred today",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_ERRORS_LAST_1_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of errors that occurred in the last 1 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_ERRORS_LAST_7_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of errors that occurred in the last 7 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_ERRORS_LAST_30_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of errors that occurred in the last 30 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_ERRORS_TOTAL,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Total number of errors",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_QUERIES_TODAY,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of queries that occurred today",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_QUERIES_LAST_1_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of queries that occurred in the last 1 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_QUERIES_LAST_7_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of queries that occurred in the last 7 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_QUERIES_LAST_30_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of queries that occurred in the last 30 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_QUERIES_TOTAL,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Total number of queries",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_VIEWS_TODAY,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of views that occurred today",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_VIEWS_LAST_1_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of views that occurred in the last 1 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_VIEWS_LAST_7_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of views that occurred in the last 7 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_VIEWS_LAST_30_DAYS,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Number of views that occurred in the last 30 days",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
{
Name: DASHBOARD_VIEWS_TOTAL,
Type: resource.ResourceTableColumnDefinition_INT64,
Description: "Total number of views",
Properties: &resource.ResourceTableColumnDefinition_Properties{
Filterable: true,
},
},
})
if namespaced == nil {
namespaced = func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) {
@@ -215,3 +335,36 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
return doc, nil
}
func DashboardFields() []string {
baseFields := []string{
DASHBOARD_LEGACY_ID,
DASHBOARD_SCHEMA_VERSION,
DASHBOARD_LINK_COUNT,
DASHBOARD_PANEL_TYPES,
DASHBOARD_DS_TYPES,
DASHBOARD_TRANSFORMATIONS,
}
return append(baseFields, UsageInsightsFields()...)
}
func UsageInsightsFields() []string {
return []string{
DASHBOARD_VIEWS_LAST_1_DAYS,
DASHBOARD_VIEWS_LAST_7_DAYS,
DASHBOARD_VIEWS_LAST_30_DAYS,
DASHBOARD_VIEWS_TODAY,
DASHBOARD_VIEWS_TOTAL,
DASHBOARD_QUERIES_LAST_1_DAYS,
DASHBOARD_QUERIES_LAST_7_DAYS,
DASHBOARD_QUERIES_LAST_30_DAYS,
DASHBOARD_QUERIES_TODAY,
DASHBOARD_QUERIES_TOTAL,
DASHBOARD_ERRORS_LAST_1_DAYS,
DASHBOARD_ERRORS_LAST_7_DAYS,
DASHBOARD_ERRORS_LAST_30_DAYS,
DASHBOARD_ERRORS_TODAY,
DASHBOARD_ERRORS_TOTAL,
}
}
pkg/storage/unified/search/testdata/manual-dashboard.json
Vendored
+155
View File
@@ -63,6 +63,111 @@
"format": "",
"description": "How many links appear on the page",
"priority": 0
},
{
"name": "errors_today",
"type": "number",
"format": "int64",
"description": "Number of errors that occurred today",
"priority": 0
},
{
"name": "errors_last_1_days",
"type": "number",
"format": "int64",
"description": "Number of errors that occurred in the last 1 days",
"priority": 0
},
{
"name": "errors_last_7_days",
"type": "number",
"format": "int64",
"description": "Number of errors that occurred in the last 7 days",
"priority": 0
},
{
"name": "errors_last_30_days",
"type": "number",
"format": "int64",
"description": "Number of errors that occurred in the last 30 days",
"priority": 0
},
{
"name": "errors_total",
"type": "number",
"format": "int64",
"description": "Total number of errors",
"priority": 0
},
{
"name": "queries_today",
"type": "number",
"format": "int64",
"description": "Number of queries that occurred today",
"priority": 0
},
{
"name": "queries_last_1_days",
"type": "number",
"format": "int64",
"description": "Number of queries that occurred in the last 1 days",
"priority": 0
},
{
"name": "queries_last_7_days",
"type": "number",
"format": "int64",
"description": "Number of queries that occurred in the last 7 days",
"priority": 0
},
{
"name": "queries_last_30_days",
"type": "number",
"format": "int64",
"description": "Number of queries that occurred in the last 30 days",
"priority": 0
},
{
"name": "queries_total",
"type": "number",
"format": "int64",
"description": "Total number of queries",
"priority": 0
},
{
"name": "views_today",
"type": "number",
"format": "int64",
"description": "Number of views that occurred today",
"priority": 0
},
{
"name": "views_last_1_days",
"type": "number",
"format": "int64",
"description": "Number of views that occurred in the last 1 days",
"priority": 0
},
{
"name": "views_last_7_days",
"type": "number",
"format": "int64",
"description": "Number of views that occurred in the last 7 days",
"priority": 0
},
{
"name": "views_last_30_days",
"type": "number",
"format": "int64",
"description": "Number of views that occurred in the last 30 days",
"priority": 0
},
{
"name": "views_total",
"type": "number",
"format": "int64",
"description": "Total number of views",
"priority": 0
}
],
"rows": [
@@ -78,6 +183,21 @@
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"object": {
@@ -102,6 +222,23 @@
null,
null,
null,
[
"timeseries"
],
40,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
100,
null,
null,
null
],
"object": {
@@ -127,6 +264,24 @@
null,
null,
null,
[
"timeseries",
"table"
],
25,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
50,
null,
null,
null
],
"object": {