public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Search: add feature flag and basic service (#45112)

Browse Source 
Co-authored-by: Artur Wierzbicki <artur@arturwierzbicki.com>
This commit is contained in:
Ryan McKinley
2022-02-15 10:26:03 -08:00
committed by GitHub
parent 11433cba97
commit d665306ad1
19 changed files with 884 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/searchV2/extract/dashboard.go
+206
View File
@@ -0,0 +1,206 @@
package extract
import (
"fmt"
"io"
jsoniter "github.com/json-iterator/go"
)
func logf(format string, a ...interface{}) {
fmt.Printf(format, a...)
}
// nolint:gocyclo
// ReadDashboard will take a byte stream and return dashboard info
func ReadDashboard(stream io.Reader, datasource DatasourceLookup) *DashboardInfo {
iter := jsoniter.Parse(jsoniter.ConfigDefault, stream, 1024)
dash := &DashboardInfo{}
for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
// Skip null values so we don't need special int handling
if iter.WhatIsNext() == jsoniter.NilValue {
iter.Skip()
continue
}
switch l1Field {
case "id":
dash.ID = iter.ReadInt64()
case "uid":
dash.UID = iter.ReadString()
case "title":
dash.Title = iter.ReadString()
case "description":
dash.Description = iter.ReadString()
case "schemaVersion":
dash.SchemaVersion = iter.ReadInt64()
case "timezone":
dash.TimeZone = iter.ReadString()
case "editable":
dash.ReadOnly = !iter.ReadBool()
case "refresh":
nxt := iter.WhatIsNext()
if nxt == jsoniter.StringValue {
dash.Refresh = iter.ReadString()
} else {
iter.Skip()
}
case "tags":
for iter.ReadArray() {
dash.Tags = append(dash.Tags, iter.ReadString())
}
case "links":
for iter.ReadArray() {
iter.Skip()
dash.LinkCount++
}
case "time":
obj, ok := iter.Read().(map[string]interface{})
if ok {
dash.TimeFrom, _ = obj["from"].(string)
dash.TimeTo, _ = obj["to"].(string)
}
case "panels":
for iter.ReadArray() {
dash.Panels = append(dash.Panels, readPanelInfo(iter))
}
case "rows":
for iter.ReadArray() {
v := iter.Read()
logf("[DASHBOARD.ROW???] id=%s // %v\n", dash.UID, v)
}
case "annotations":
for sub := iter.ReadObject(); sub != ""; sub = iter.ReadObject() {
if sub == "list" {
for iter.ReadArray() {
v := iter.Read()
logf("[dash.anno] %v\n", v)
}
} else {
iter.Skip()
}
}
case "templating":
for sub := iter.ReadObject(); sub != ""; sub = iter.ReadObject() {
if sub == "list" {
for iter.ReadArray() {
for k := iter.ReadObject(); k != ""; k = iter.ReadObject() {
if k == "name" {
dash.TemplateVars = append(dash.TemplateVars, iter.ReadString())
} else {
iter.Skip()
}
}
}
} else {
iter.Skip()
}
}
// Ignore these properties
case "timepicker":
fallthrough
case "version":
fallthrough
case "iteration":
iter.Skip()
default:
v := iter.Read()
logf("[DASHBOARD] support key: %s / %v\n", l1Field, v)
}
}
if dash.UID == "" {
logf("All dashbaords should have a UID defined")
}
return dash
}
// will always return strings for now
func readPanelInfo(iter *jsoniter.Iterator) PanelInfo {
panel := PanelInfo{}
for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
// Skip null values so we don't need special int handling
if iter.WhatIsNext() == jsoniter.NilValue {
iter.Skip()
continue
}
switch l1Field {
case "id":
panel.ID = iter.ReadInt64()
case "type":
panel.Type = iter.ReadString()
case "title":
panel.Title = iter.ReadString()
case "description":
panel.Description = iter.ReadString()
case "pluginVersion":
panel.PluginVersion = iter.ReadString() // since 7x (the saved version for the plugin model)
case "datasource":
v := iter.Read()
logf(">>Panel.datasource = %v\n", v) // string or object!!!
case "targets":
for iter.ReadArray() {
v := iter.Read()
logf("[Panel.TARGET] %v\n", v)
}
case "transformations":
for iter.ReadArray() {
for sub := iter.ReadObject(); sub != ""; sub = iter.ReadObject() {
if sub == "id" {
panel.Transformations = append(panel.Transformations, iter.ReadString())
} else {
iter.Skip()
}
}
}
// Rows have nested panels
case "panels":
for iter.ReadArray() {
panel.Collapsed = append(panel.Collapsed, readPanelInfo(iter))
}
case "options":
fallthrough
case "gridPos":
fallthrough
case "fieldConfig":
iter.Skip()
default:
v := iter.Read()
logf("[PANEL] support key: %s / %v\n", l1Field, v)
}
}
return panel
}
pkg/services/searchV2/extract/dashboard_test.go
+48
View File
@@ -0,0 +1,48 @@
package extract
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReadDashboard(t *testing.T) {
inputs := []string{
"all-panels.json",
"panel-graph/graph-shared-tooltips.json",
}
// key will allow name or uid
ds := func(key string) *DatasourceInfo {
return nil // TODO!
}
for _, input := range inputs {
// nolint:gosec
// We can ignore the gosec G304 warning because this is a test with hardcoded input values
f, err := os.Open("../../../../devenv/dev-dashboards/" + input)
require.NoError(t, err)
dash := ReadDashboard(f, ds)
out, err := json.MarshalIndent(dash, "", " ")
require.NoError(t, err)
update := false
savedPath := "testdata/" + filepath.Base(input)
saved, err := os.ReadFile(savedPath)
if err != nil {
update = true
assert.NoError(t, err)
} else if !assert.JSONEq(t, string(saved), string(out)) {
update = true
}
if update {
_ = os.WriteFile(savedPath, out, 0600)
}
}
}
pkg/services/searchV2/extract/testdata/all-panels.json
Vendored
+128
View File
@@ -0,0 +1,128 @@
{
"uid": "n1jR8vnnz",
"title": "Panel tests - All panels",
"tags": [
"gdev",
"panel-tests",
"all-panels"
],
"templateVars": [
"query0",
"query1",
"text"
],
"panels": [
{
"id": 34,
"title": "",
"type": "text",
"pluginVersion": "8.1.0-pre"
},
{
"id": 35,
"title": "",
"type": "text",
"pluginVersion": "8.1.0-pre"
},
{
"id": 32,
"title": "Row title",
"type": "row"
},
{
"id": 41,
"title": "State timeline",
"type": "state-timeline"
},
{
"id": 62,
"title": "Size, color mapped to different fields + share view",
"type": "geomap"
},
{
"id": 4,
"title": "Histogram",
"type": "histogram"
},
{
"id": 28,
"title": "Logs",
"type": "logs"
},
{
"id": 8,
"title": "Dashboard list",
"type": "dashlist",
"pluginVersion": "8.1.0-pre"
},
{
"id": 30,
"title": "Panel list",
"type": "pluginlist"
},
{
"id": 6,
"title": "Alert list",
"type": "alertlist"
},
{
"id": 26,
"title": "Heatmap",
"type": "heatmap"
},
{
"id": 20,
"title": "Bar gauge",
"type": "bargauge",
"pluginVersion": "8.1.0-pre"
},
{
"id": 24,
"title": "Pie chart",
"type": "piechart"
},
{
"id": 18,
"title": "Gauge",
"type": "gauge",
"pluginVersion": "8.1.0-pre"
},
{
"id": 22,
"title": "Tabel",
"type": "table",
"pluginVersion": "8.1.0-pre"
},
{
"id": 10,
"title": "Annotation list",
"type": "annolist"
},
{
"id": 16,
"title": "Stat",
"type": "stat",
"pluginVersion": "8.1.0-pre"
},
{
"id": 2,
"title": "Graph NG",
"type": "timeseries"
},
{
"id": 14,
"title": "Bar chart",
"type": "barchart"
},
{
"id": 12,
"title": "News panel",
"type": "news"
}
],
"schemaVersion": 33,
"linkCount": 2,
"timeFrom": "now-6h",
"timeTo": "now",
"timezone": ""
}
pkg/services/searchV2/extract/testdata/graph-shared-tooltips.json
Vendored
+68
View File
@@ -0,0 +1,68 @@
{
"uid": "TX2VU59MZ",
"title": "Panel Tests - shared tooltips",
"tags": [
"gdev",
"panel-tests",
"graph-ng"
],
"panels": [
{
"id": 4,
"title": "two units",
"type": "timeseries",
"pluginVersion": "7.5.0-pre"
},
{
"id": 13,
"title": "Speed vs Temperature (XY)",
"type": "xychart",
"pluginVersion": "7.5.0-pre",
"transformations": [
"seriesToColumns",
"organize"
]
},
{
"id": 2,
"title": "Cursor info",
"type": "debug",
"pluginVersion": "7.5.0-pre"
},
{
"id": 5,
"title": "Only temperature",
"type": "timeseries",
"pluginVersion": "7.5.0-pre"
},
{
"id": 9,
"title": "Only Speed",
"type": "timeseries",
"pluginVersion": "7.5.0-pre"
},
{
"id": 11,
"title": "Panel Title",
"type": "timeseries",
"pluginVersion": "7.5.0-pre"
},
{
"id": 8,
"title": "flot panel (temperature)",
"type": "graph",
"pluginVersion": "7.5.0-pre"
},
{
"id": 10,
"title": "flot panel (no units)",
"type": "graph",
"pluginVersion": "7.5.0-pre"
}
],
"schemaVersion": 28,
"linkCount": 0,
"timeFrom": "2020-09-14T16:13:20.000Z",
"timeTo": "2020-09-15T20:00:00.000Z",
"timezone": ""
}
pkg/services/searchV2/extract/types.go
+45
View File
@@ -0,0 +1,45 @@
package extract
type DatasourceLookup = func(key string) *DatasourceInfo
type DatasourceInfo struct {
UID string `json:"uid"`
Name string `json:"name"`
Type string `json:"type"` // plugin name
Version string `json:"version"`
Access string `json:"access,omitempty"` // proxy, direct, or empty
}
type PanelInfo struct {
ID int64 `json:"id"`
Title string `json:"title"`
Description string `json:"description,omitempty"`
Type string `json:"type,omitempty"` // PluginID
PluginVersion string `json:"pluginVersion,omitempty"`
Datasource []string `json:"datasource,omitempty"` // UIDs
DatasourceType []string `json:"datasourceType,omitempty"` // PluginIDs
Transformations []string `json:"transformations,omitempty"` // ids of the transformation steps
// Rows define panels as sub objects
Collapsed []PanelInfo `json:"collapsed,omitempty"`
}
type DashboardInfo struct {
ID int64 `json:"id,omitempty"`
UID string `json:"uid,omitempty"`
Path string `json:"path,omitempty"`
Title string `json:"title"`
Description string `json:"description,omitempty"`
Tags []string `json:"tags"` // UIDs
Datasource []string `json:"datasource,omitempty"` // UIDs
DatasourceType []string `json:"datasourceType,omitempty"` // PluginIDs
TemplateVars []string `json:"templateVars,omitempty"` // the keys used
Panels []PanelInfo `json:"panels"` // nesed documents
SchemaVersion int64 `json:"schemaVersion"`
LinkCount int64 `json:"linkCount"`
TimeFrom string `json:"timeFrom"`
TimeTo string `json:"timeTo"`
TimeZone string `json:"timezone"`
Refresh string `json:"refresh,omitempty"`
ReadOnly bool `json:"readOnly,omitempty"` // editable = false
}
pkg/services/searchV2/service.go
+234
View File
@@ -0,0 +1,234 @@
package searchV2
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/searchV2/extract"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
type StandardSearchService struct {
sql *sqlstore.SQLStore
}
func ProvideService(sql *sqlstore.SQLStore) SearchService {
return &StandardSearchService{
sql: sql,
}
}
type dashMeta struct {
id int64
is_folder bool
folder_id int64
created time.Time
updated time.Time
dash *extract.DashboardInfo
}
func (s *StandardSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse {
rsp := &backend.DataResponse{}
if user == nil || user.Role != string(models.ROLE_ADMIN) {
rsp.Error = fmt.Errorf("search is only supported for admin users while in early development")
return rsp
}
// Load and parse all dashboards for given orgId
dash, err := loadDashboards(ctx, orgId, s.sql)
if err != nil {
rsp.Error = err
return rsp
}
rsp.Frames = metaToFrame(dash)
return rsp
}
type dashDataQueryResult struct {
Id int64
IsFolder bool `xorm:"is_folder"`
FolderID int64 `xorm:"folder_id"`
Data []byte
Created time.Time
Updated time.Time
}
func loadDashboards(ctx context.Context, orgID int64, sql *sqlstore.SQLStore) ([]dashMeta, error) {
meta := make([]dashMeta, 0, 200)
// key will allow name or uid
lookup := func(key string) *extract.DatasourceInfo {
return nil // TODO!
}
err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
rows := make([]*dashDataQueryResult, 0)
sess.Table("dashboard").
Where("org_id = ?", orgID).
Cols("id", "is_folder", "folder_id", "data", "created", "updated")
err := sess.Find(&rows)
if err != nil {
return err
}
for _, row := range rows {
dash := extract.ReadDashboard(bytes.NewReader(row.Data), lookup)
meta = append(meta, dashMeta{
id: row.Id,
is_folder: row.IsFolder,
folder_id: row.FolderID,
created: row.Created,
updated: row.Updated,
dash: dash,
})
}
return nil
})
return meta, err
}
type simpleCounter struct {
values map[string]int64
}
func (c *simpleCounter) add(key string) {
v, ok := c.values[key]
if !ok {
v = 0
}
c.values[key] = v + 1
}
func (c *simpleCounter) toFrame(name string) *data.Frame {
key := data.NewFieldFromFieldType(data.FieldTypeString, 0)
val := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
for k, v := range c.values {
key.Append(k)
val.Append(v)
}
return data.NewFrame(name, key, val)
}
// UGLY... but helpful for now
func metaToFrame(meta []dashMeta) data.Frames {
folderID := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
folderUID := data.NewFieldFromFieldType(data.FieldTypeString, 0)
folderName := data.NewFieldFromFieldType(data.FieldTypeString, 0)
folderID.Name = "ID"
folderUID.Name = "UID"
folderName.Name = "Name"
dashID := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
dashUID := data.NewFieldFromFieldType(data.FieldTypeString, 0)
dashFolderID := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
dashName := data.NewFieldFromFieldType(data.FieldTypeString, 0)
dashDescr := data.NewFieldFromFieldType(data.FieldTypeString, 0)
dashCreated := data.NewFieldFromFieldType(data.FieldTypeTime, 0)
dashUpdated := data.NewFieldFromFieldType(data.FieldTypeTime, 0)
dashSchemaVersion := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
dashTags := data.NewFieldFromFieldType(data.FieldTypeNullableString, 0)
dashID.Name = "ID"
dashUID.Name = "UID"
dashFolderID.Name = "FolderID"
dashName.Name = "Name"
dashDescr.Name = "Description"
dashTags.Name = "Tags"
dashSchemaVersion.Name = "SchemaVersion"
dashCreated.Name = "Created"
dashUpdated.Name = "Updated"
dashTags.Config = &data.FieldConfig{
Custom: map[string]interface{}{
// Table panel default styling
"displayMode": "json-view",
},
}
panelDashID := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
panelID := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
panelName := data.NewFieldFromFieldType(data.FieldTypeString, 0)
panelDescr := data.NewFieldFromFieldType(data.FieldTypeString, 0)
panelType := data.NewFieldFromFieldType(data.FieldTypeString, 0)
panelDashID.Name = "DashboardID"
panelID.Name = "ID"
panelName.Name = "Name"
panelDescr.Name = "Description"
panelType.Name = "Type"
panelTypeCounter := simpleCounter{
values: make(map[string]int64, 30),
}
schemaVersionCounter := simpleCounter{
values: make(map[string]int64, 30),
}
var tags *string
for _, row := range meta {
if row.is_folder {
folderID.Append(row.id)
folderUID.Append(row.dash.UID)
folderName.Append(row.dash.Title)
continue
}
dashID.Append(row.id)
dashUID.Append(row.dash.UID)
dashFolderID.Append(row.folder_id)
dashName.Append(row.dash.Title)
dashDescr.Append(row.dash.Title)
dashSchemaVersion.Append(row.dash.SchemaVersion)
dashCreated.Append(row.created)
dashUpdated.Append(row.updated)
// stats
schemaVersionCounter.add(strconv.FormatInt(row.dash.SchemaVersion, 10))
// Send tags as JSON array
tags = nil
if len(row.dash.Tags) > 0 {
b, err := json.Marshal(row.dash.Tags)
if err == nil {
s := string(b)
tags = &s
}
}
dashTags.Append(tags)
// Row for each panel
for _, panel := range row.dash.Panels {
panelDashID.Append(row.id)
panelID.Append(panel.ID)
panelName.Append(panel.Title)
panelDescr.Append(panel.Description)
panelType.Append(panel.Type)
panelTypeCounter.add(panel.Type)
}
}
return data.Frames{
data.NewFrame("folders", folderID, folderUID, folderName),
data.NewFrame("dashboards", dashID, dashUID, dashFolderID, dashName, dashDescr, dashTags, dashSchemaVersion, dashCreated, dashUpdated),
data.NewFrame("panels", panelDashID, panelID, panelName, panelDescr, panelType),
panelTypeCounter.toFrame("panel-type-counts"),
schemaVersionCounter.toFrame("schema-version-counts"),
}
}
pkg/services/searchV2/stub.go
+30
View File
@@ -0,0 +1,30 @@
package searchV2
import (
"context"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
)
type stubSearchService struct {
}
func NewStubSearchService() SearchService {
return &stubSearchService{}
}
func (s *stubSearchService) DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse {
rsp := &backend.DataResponse{}
// dashboards
fid := data.NewFieldFromFieldType(data.FieldTypeInt64, 0)
uid := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fid.Append(int64(2))
uid.Append("hello")
rsp.Frames = append(rsp.Frames, data.NewFrame("dasboards", fid, uid))
return rsp
}
pkg/services/searchV2/types.go
+15
View File
@@ -0,0 +1,15 @@
package searchV2
import (
"context"
"github.com/grafana/grafana-plugin-sdk-go/backend"
)
type DashboardQuery struct {
Query string
}
type SearchService interface {
DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse
}