public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Public Dashboards: Add Expressions Support (#54336)

Browse Source 
Adds support for expressions with public dashboards
This commit is contained in:
owensmallwood
2022-08-31 09:11:10 -06:00
committed by GitHub
parent 76ea0b15ae
commit f4bbce15a0
7 changed files with 197 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/query/models/dashboard_queries.go
+99
View File
@@ -0,0 +1,99 @@
package query
import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/expr"
)
func GetUniqueDashboardDatasourceUids(dashboard *simplejson.Json) []string {
var datasourceUids []string
exists := map[string]bool{}
for _, panelObj := range dashboard.Get("panels").MustArray() {
panel := simplejson.NewFromAny(panelObj)
uid := GetDataSourceUidFromJson(panel)
// if uid is for a mixed datasource, get the datasource uids from the targets
if uid == "-- Mixed --" {
for _, target := range panel.Get("targets").MustArray() {
target := simplejson.NewFromAny(target)
datasourceUid := target.Get("datasource").Get("uid").MustString()
if _, ok := exists[datasourceUid]; !ok {
datasourceUids = append(datasourceUids, datasourceUid)
exists[datasourceUid] = true
}
}
} else {
if _, ok := exists[uid]; !ok {
datasourceUids = append(datasourceUids, uid)
exists[uid] = true
}
}
}
return datasourceUids
}
func GroupQueriesByPanelId(dashboard *simplejson.Json) map[int64][]*simplejson.Json {
result := make(map[int64][]*simplejson.Json)
for _, panelObj := range dashboard.Get("panels").MustArray() {
panel := simplejson.NewFromAny(panelObj)
var panelQueries []*simplejson.Json
for _, queryObj := range panel.Get("targets").MustArray() {
query := simplejson.NewFromAny(queryObj)
// if query target has no datasource, set it to have the datasource on the panel
if _, ok := query.CheckGet("datasource"); !ok {
uid := GetDataSourceUidFromJson(panel)
datasource := map[string]interface{}{"type": "public-ds", "uid": uid}
query.Set("datasource", datasource)
}
panelQueries = append(panelQueries, query)
}
result[panel.Get("id").MustInt64()] = panelQueries
}
return result
}
func HasExpressionQuery(queries []*simplejson.Json) bool {
for _, query := range queries {
uid := GetDataSourceUidFromJson(query)
if expr.IsDataSource(uid) {
return true
}
}
return false
}
func GroupQueriesByDataSource(queries []*simplejson.Json) (result [][]*simplejson.Json) {
byDataSource := make(map[string][]*simplejson.Json)
for _, query := range queries {
uid := GetDataSourceUidFromJson(query)
byDataSource[uid] = append(byDataSource[uid], query)
}
for _, queries := range byDataSource {
result = append(result, queries)
}
return
}
func GetDataSourceUidFromJson(query *simplejson.Json) string {
uid := query.Get("datasource").Get("uid").MustString()
// before 8.3 special types could be sent as datasource (expr)
if uid == "" {
uid = query.Get("datasource").MustString()
}
return uid
}
pkg/services/query/models/dashboard_queries_test.go
+504
View File
@@ -0,0 +1,504 @@
package query
import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/stretchr/testify/require"
)
const (
dashboardWithNoQueries = `
{
"panels": [
{
"id": 2,
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 35
}`
dashboardWithTargetsWithNoDatasources = `
{
"panels": [
{
"id": 2,
"datasource": {
"type": "postgres",
"uid": "abc123"
},
"targets": [
{
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"exemplar": true,
"expr": "query2",
"interval": "",
"legendFormat": "",
"refId": "B"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 35
}`
dashboardWithQueries = `
{
"panels": [
{
"id": 2,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "promds2"
},
"exemplar": true,
"expr": "query2",
"interval": "",
"legendFormat": "",
"refId": "B"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 35
}`
dashboardWithQueriesAndExpression = `
{
"panels": [
{
"id": 2,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "$A + 1",
"hide": false,
"refId": "EXPRESSION",
"type": "math"
},
{
"datasource": {
"type": "prometheus",
"uid": "promds2"
},
"exemplar": true,
"expr": "query2",
"interval": "",
"legendFormat": "",
"refId": "B"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 35
}`
dashboardWithMixedDatasource = `
{
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "-- Mixed --"
},
"id": 1,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "abc123"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"id": 2,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"id": 3,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 35
}`
dashboardWithDuplicateDatasources = `
{
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "abc123"
},
"id": 1,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "abc123"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"id": 2,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"id": 3,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 35
}`
oldStyleDashboard = `
{
"panels": [
{
"datasource": "_yxMP8Ynk",
"id": 2,
"targets": [
{
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"schemaVersion": 21
}`
)
func TestGetUniqueDashboardDatasourceUids(t *testing.T) {
t.Run("can get unique datasource ids from dashboard", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithDuplicateDatasources))
require.NoError(t, err)
uids := GetUniqueDashboardDatasourceUids(json)
require.Len(t, uids, 2)
require.Equal(t, "abc123", uids[0])
require.Equal(t, "_yxMP8Ynk", uids[1])
})
t.Run("can get unique datasource ids from dashboard with a mixed datasource", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithMixedDatasource))
require.NoError(t, err)
uids := GetUniqueDashboardDatasourceUids(json)
require.Len(t, uids, 2)
require.Equal(t, "abc123", uids[0])
require.Equal(t, "_yxMP8Ynk", uids[1])
})
t.Run("can get no datasource uids from empty dashboard", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(`{"panels": {}}`))
require.NoError(t, err)
uids := GetUniqueDashboardDatasourceUids(json)
require.Len(t, uids, 0)
})
}
func TestHasExpressionQuery(t *testing.T) {
t.Run("will return true when expression query exists", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithQueriesAndExpression))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
panelId := int64(2)
require.True(t, HasExpressionQuery(queries[panelId]))
})
t.Run("will return false when no expression query exists", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithMixedDatasource))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
panelId := int64(2)
require.False(t, HasExpressionQuery(queries[panelId]))
})
}
func TestGroupQueriesByPanelId(t *testing.T) {
t.Run("can extract queries from dashboard with panel datasource string that has no datasource on panel targets", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(oldStyleDashboard))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
panelId := int64(2)
queriesByDatasource := GroupQueriesByDataSource(queries[panelId])
require.Len(t, queriesByDatasource[0], 1)
})
t.Run("can extract queries from dashboard with panel json datasource that has no datasource on panel targets", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithTargetsWithNoDatasources))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
panelId := int64(2)
queriesByDatasource := GroupQueriesByDataSource(queries[panelId])
require.Len(t, queriesByDatasource[0], 2)
})
t.Run("can extract no queries from empty dashboard", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(`{"panels": {}}`))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
require.Len(t, queries, 0)
})
t.Run("can extract no queries from empty panel", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithNoQueries))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
require.Len(t, queries, 1)
require.Contains(t, queries, int64(2))
require.Len(t, queries[2], 0)
})
t.Run("can extract queries from panels", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(dashboardWithQueries))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
require.Len(t, queries, 1)
require.Contains(t, queries, int64(2))
require.Len(t, queries[2], 2)
query, err := queries[2][0].MarshalJSON()
require.NoError(t, err)
require.JSONEq(t, `{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}`, string(query))
query, err = queries[2][1].MarshalJSON()
require.NoError(t, err)
require.JSONEq(t, `{
"datasource": {
"type": "prometheus",
"uid": "promds2"
},
"exemplar": true,
"expr": "query2",
"interval": "",
"legendFormat": "",
"refId": "B"
}`, string(query))
})
t.Run("can extract queries from old-style panels", func(t *testing.T) {
json, err := simplejson.NewJson([]byte(oldStyleDashboard))
require.NoError(t, err)
queries := GroupQueriesByPanelId(json)
require.Len(t, queries, 1)
require.Contains(t, queries, int64(2))
require.Len(t, queries[2], 1)
query, err := queries[2][0].MarshalJSON()
require.NoError(t, err)
require.JSONEq(t, `{
"datasource": {
"uid": "_yxMP8Ynk",
"type": "public-ds"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}`, string(query))
})
}
func TestGroupQueriesByDataSource(t *testing.T) {
t.Run("can divide queries by datasource", func(t *testing.T) {
queries := []*simplejson.Json{
simplejson.MustJson([]byte(`{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}`)),
simplejson.MustJson([]byte(`{
"datasource": {
"type": "prometheus",
"uid": "promds2"
},
"exemplar": true,
"expr": "query2",
"interval": "",
"legendFormat": "",
"refId": "B"
}`)),
}
queriesByDatasource := GroupQueriesByDataSource(queries)
require.Len(t, queriesByDatasource, 2)
require.Contains(t, queriesByDatasource, []*simplejson.Json{simplejson.MustJson([]byte(`{
"datasource": {
"type": "prometheus",
"uid": "_yxMP8Ynk"
},
"exemplar": true,
"expr": "go_goroutines{job=\"$job\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}`))})
require.Contains(t, queriesByDatasource, []*simplejson.Json{simplejson.MustJson([]byte(`{
"datasource": {
"type": "prometheus",
"uid": "promds2"
},
"exemplar": true,
"expr": "query2",
"interval": "",
"legendFormat": "",
"refId": "B"
}`))})
})
}
pkg/services/query/query.go
+5 -2
View File
@@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana/pkg/plugins/adapters"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/oauthtoken"
queryModels "github.com/grafana/grafana/pkg/services/query/models"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/grafanads"
@@ -80,13 +81,15 @@ func (s *Service) QueryData(ctx context.Context, user *user.SignedInUser, skipCa
// QueryData can process queries and return query responses.
func (s *Service) QueryDataMultipleSources(ctx context.Context, user *user.SignedInUser, skipCache bool, reqDTO dtos.MetricRequest, handleExpressions bool) (*backend.QueryDataResponse, error) {
byDataSource := models.GroupQueriesByDataSource(reqDTO.Queries)
byDataSource := queryModels.GroupQueriesByDataSource(reqDTO.Queries)
if len(byDataSource) == 1 {
// The expression service will handle mixed datasources, so we don't need to group them when an expression is present.
if queryModels.HasExpressionQuery(reqDTO.Queries) || len(byDataSource) == 1 {
return s.QueryData(ctx, user, skipCache, reqDTO, handleExpressions)
} else {
resp := backend.NewQueryDataResponse()
// create new reqDTO with only the queries for that datasource
for _, queries := range byDataSource {
subDTO := reqDTO.CloneWithQueries(queries)
pkg/services/query/query_test.go
+102 -2
View File
@@ -2,10 +2,12 @@ package query_test
import (
"context"
"errors"
"net/http"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/expr"
"github.com/stretchr/testify/require"
"golang.org/x/oauth2"
@@ -15,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/plugins"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/datasources"
fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
dsSvc "github.com/grafana/grafana/pkg/services/datasources/service"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/query"
@@ -25,6 +28,92 @@ import (
"github.com/grafana/grafana/pkg/services/user"
)
func TestQueryDataMultipleSources(t *testing.T) {
t.Run("can query multiple datasources", func(t *testing.T) {
tc := setup(t)
query1, err := simplejson.NewJson([]byte(`
{
"datasource": {
"type": "mysql",
"uid": "ds1"
}
}
`))
require.NoError(t, err)
query2, err := simplejson.NewJson([]byte(`
{
"datasource": {
"type": "mysql",
"uid": "ds2"
}
}
`))
require.NoError(t, err)
queries := []*simplejson.Json{query1, query2}
reqDTO := dtos.MetricRequest{
From: "2022-01-01",
To: "2022-01-02",
Queries: queries,
Debug: false,
PublicDashboardAccessToken: "abc123",
HTTPRequest: nil,
}
_, err = tc.queryService.QueryDataMultipleSources(context.Background(), nil, true, reqDTO, false)
require.NoError(t, err)
})
t.Run("can query multiple datasources with an expression present", func(t *testing.T) {
tc := setup(t)
query1, err := simplejson.NewJson([]byte(`
{
"datasource": {
"type": "mysql",
"uid": "ds1"
}
}
`))
require.NoError(t, err)
query2, err := simplejson.NewJson([]byte(`
{
"datasource": {
"type": "mysql",
"uid": "ds2"
}
}
`))
require.NoError(t, err)
query3, err := simplejson.NewJson([]byte(`
{
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "$A + 1",
"hide": false,
"refId": "EXPRESSION",
"type": "math"
}
`))
require.NoError(t, err)
queries := []*simplejson.Json{query1, query2, query3}
reqDTO := dtos.MetricRequest{
From: "2022-01-01",
To: "2022-01-02",
Queries: queries,
Debug: false,
PublicDashboardAccessToken: "abc123",
HTTPRequest: nil,
}
_, err = tc.queryService.QueryDataMultipleSources(context.Background(), nil, true, reqDTO, false)
require.NoError(t, err)
})
}
func TestQueryData(t *testing.T) {
t.Run("it auth custom headers to the request", func(t *testing.T) {
token := &oauth2.Token{
@@ -96,6 +185,11 @@ func setup(t *testing.T) *testContext {
ss := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
ssvc := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
ds := dsSvc.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
fakeDatasourceService := &fakeDatasources.FakeDataSourceService{
DataSources: nil,
SimulatePluginFailure: false,
}
exprService := expr.ProvideService(nil, pc, fakeDatasourceService)
return &testContext{
pluginContext: pc,
@@ -103,7 +197,7 @@ func setup(t *testing.T) *testContext {
dataSourceCache: dc,
oauthTokenService: tc,
pluginRequestValidator: rv,
queryService: query.ProvideService(nil, dc, nil, rv, ds, pc, tc),
queryService: query.ProvideService(nil, dc, exprService, rv, ds, pc, tc),
}
}
@@ -167,5 +261,11 @@ type fakePluginClient struct {
func (c *fakePluginClient) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
c.req = req
return nil, nil
// If an expression query ends up getting directly queried, we want it to return an error in our test.
if req.PluginContext.PluginID == "__expr__" {
return nil, errors.New("cant query an expression datasource")
}
return &backend.QueryDataResponse{Responses: make(backend.Responses)}, nil
}