Introduce TSDB service (#31520)

* Introduce TSDB service

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
Co-authored-by: Will Browne <will.browne@grafana.com>
Co-authored-by: Torkel Ödegaard <torkel@grafana.org>
Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
Co-authored-by: Zoltán Bedi <zoltan.bedi@gmail.com>
This commit is contained in:
Arve Knudsen
2021-03-08 07:02:49 +01:00
committed by GitHub
co-authored by Erik Sundell Will Browne Torkel Ödegaard Will Browne Zoltán Bedi
parent c899bf3592
commit b79e61656a
203 changed files with 5269 additions and 4776 deletions
@@ -18,8 +18,8 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
@@ -49,20 +49,22 @@ type ApplicationInsightsQuery struct {
aggregation string
}
func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []*tsdb.Query, timeRange *tsdb.TimeRange) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Context,
originalQueries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
for _, query := range queries {
queryRes, err := e.executeQuery(ctx, query)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
result.Results[query.RefID] = queryRes
}
@@ -70,7 +72,8 @@ func (e *ApplicationInsightsDatasource) executeTimeSeriesQuery(ctx context.Conte
return result, nil
}
func (e *ApplicationInsightsDatasource) buildQueries(queries []*tsdb.Query, timeRange *tsdb.TimeRange) ([]*ApplicationInsightsQuery, error) {
func (e *ApplicationInsightsDatasource) buildQueries(queries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) ([]*ApplicationInsightsQuery, error) {
applicationInsightsQueries := []*ApplicationInsightsQuery{}
startTime, err := timeRange.ParseFrom()
if err != nil {
@@ -100,7 +103,7 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []*tsdb.Query, time
timeGrain := insightsJSONModel.TimeGrain
timeGrains := insightsJSONModel.AllowedTimeGrainsMs
if timeGrain == "auto" {
timeGrain, err = setAutoTimeGrain(query.IntervalMs, timeGrains)
timeGrain, err = setAutoTimeGrain(query.IntervalMS, timeGrains)
if err != nil {
return nil, err
}
@@ -122,7 +125,7 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []*tsdb.Query, time
params.Add("segment", strings.Join(insightsJSONModel.Dimensions, ","))
}
applicationInsightsQueries = append(applicationInsightsQueries, &ApplicationInsightsQuery{
RefID: query.RefId,
RefID: query.RefID,
ApiURL: azureURL,
Params: params,
Alias: insightsJSONModel.Alias,
@@ -136,8 +139,9 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []*tsdb.Query, time
return applicationInsightsQueries, nil
}
func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query *ApplicationInsightsQuery) (*tsdb.QueryResult, error) {
queryResult := &tsdb.QueryResult{Meta: simplejson.New(), RefId: query.RefID}
func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query *ApplicationInsightsQuery) (
plugins.DataQueryResult, error) {
queryResult := plugins.DataQueryResult{Meta: simplejson.New(), RefID: query.RefID}
req, err := e.createRequest(ctx, e.dsInfo)
if err != nil {
@@ -178,18 +182,18 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query
}
}()
if err != nil {
return nil, err
return plugins.DataQueryResult{}, err
}
if res.StatusCode/100 != 2 {
azlog.Debug("Request failed", "status", res.Status, "body", string(body))
return nil, fmt.Errorf("request failed, status: %s", res.Status)
return plugins.DataQueryResult{}, fmt.Errorf("request failed, status: %s", res.Status)
}
mr := MetricsResult{}
err = json.Unmarshal(body, &mr)
if err != nil {
return nil, err
return plugins.DataQueryResult{}, err
}
frame, err := InsightsMetricsResultToFrame(mr, query.metricName, query.aggregation, query.dimensions)
@@ -200,13 +204,13 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query
applyInsightsMetricAlias(frame, query.Alias)
queryResult.Dataframes = tsdb.NewDecodedDataFrames(data.Frames{frame})
queryResult.Dataframes = plugins.NewDecodedDataFrames(data.Frames{frame})
return queryResult, nil
}
func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) {
// find plugin
plugin, ok := plugins.DataSources[dsInfo.Type]
plugin, ok := manager.DataSources[dsInfo.Type]
if !ok {
return nil, errors.New("unable to find datasource plugin Azure Application Insights")
}
@@ -239,7 +243,8 @@ func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInf
return req, nil
}
func (e *ApplicationInsightsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (*plugins.AppPluginRoute, string, error) {
func (e *ApplicationInsightsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (
*plugins.AppPluginRoute, string, error) {
pluginRouteName := "appinsights"
if cloudName == "chinaazuremonitor" {
@@ -247,7 +252,6 @@ func (e *ApplicationInsightsDatasource) getPluginRoute(plugin *plugins.DataSourc
}
var pluginRoute *plugins.AppPluginRoute
for _, route := range plugin.Routes {
if route.Path == pluginRouteName {
pluginRoute = route
@@ -11,7 +11,6 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/stretchr/testify/require"
. "github.com/smartystreets/goconvey/convey"
@@ -23,12 +22,12 @@ func TestApplicationInsightsDatasource(t *testing.T) {
Convey("Parse queries from frontend and build AzureMonitor API queries", func() {
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
tsdbQuery := &tsdb.TsdbQuery{
TimeRange: &tsdb.TimeRange{
tsdbQuery := plugins.DataQuery{
TimeRange: &plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
Queries: []*tsdb.Query{
Queries: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
@@ -43,13 +42,13 @@ func TestApplicationInsightsDatasource(t *testing.T) {
"queryType": "Application Insights",
},
}),
RefId: "A",
IntervalMs: 1234,
RefID: "A",
IntervalMS: 1234,
},
},
}
Convey("and is a normal query", func() {
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange)
So(err, ShouldBeNil)
So(len(queries), ShouldEqual, 1)
@@ -74,9 +73,9 @@ func TestApplicationInsightsDatasource(t *testing.T) {
"queryType": "Application Insights",
},
})
tsdbQuery.Queries[0].IntervalMs = 400000
tsdbQuery.Queries[0].IntervalMS = 400000
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange)
So(err, ShouldBeNil)
So(queries[0].Params["interval"][0], ShouldEqual, "PT15M")
@@ -94,9 +93,9 @@ func TestApplicationInsightsDatasource(t *testing.T) {
"allowedTimeGrainsMs": []int64{60000, 300000},
},
})
tsdbQuery.Queries[0].IntervalMs = 400000
tsdbQuery.Queries[0].IntervalMS = 400000
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange)
So(err, ShouldBeNil)
So(queries[0].Params["interval"][0], ShouldEqual, "PT5M")
@@ -116,7 +115,7 @@ func TestApplicationInsightsDatasource(t *testing.T) {
},
})
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange)
So(err, ShouldBeNil)
So(queries[0].Target, ShouldEqual, "aggregation=Average&filter=blob+eq+%27%2A%27&interval=PT1M&segment=blob&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z")
@@ -136,7 +135,7 @@ func TestApplicationInsightsDatasource(t *testing.T) {
},
})
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange)
So(err, ShouldBeNil)
So(queries[0].Target, ShouldEqual, "aggregation=Average&interval=PT1M&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z")
@@ -17,8 +17,8 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
@@ -45,14 +45,15 @@ type AzureLogAnalyticsQuery struct {
// 1. build the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format
func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []*tsdb.Query, timeRange *tsdb.TimeRange) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
for _, query := range queries {
@@ -62,7 +63,8 @@ func (e *AzureLogAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context
return result, nil
}
func (e *AzureLogAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRange *tsdb.TimeRange) ([]*AzureLogAnalyticsQuery, error) {
func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) ([]*AzureLogAnalyticsQuery, error) {
azureLogAnalyticsQueries := []*AzureLogAnalyticsQuery{}
for _, query := range queries {
@@ -97,7 +99,7 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRa
params.Add("query", rawQuery)
azureLogAnalyticsQueries = append(azureLogAnalyticsQueries, &AzureLogAnalyticsQuery{
RefID: query.RefId,
RefID: query.RefID,
ResultFormat: resultFormat,
URL: apiURL,
Model: query.Model,
@@ -109,10 +111,11 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRa
return azureLogAnalyticsQueries, nil
}
func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *AzureLogAnalyticsQuery, queries []*tsdb.Query, timeRange *tsdb.TimeRange) *tsdb.QueryResult {
queryResult := &tsdb.QueryResult{RefId: query.RefID}
func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *AzureLogAnalyticsQuery,
queries []plugins.DataSubQuery, timeRange plugins.DataTimeRange) plugins.DataQueryResult {
queryResult := plugins.DataQueryResult{RefID: query.RefID}
queryResultErrorWithExecuted := func(err error) *tsdb.QueryResult {
queryResultErrorWithExecuted := func(err error) plugins.DataQueryResult {
queryResult.Error = err
frames := data.Frames{
&data.Frame{
@@ -122,7 +125,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
},
},
}
queryResult.Dataframes = tsdb.NewDecodedDataFrames(frames)
queryResult.Dataframes = plugins.NewDecodedDataFrames(frames)
return queryResult
}
@@ -193,7 +196,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
}
}
frames := data.Frames{frame}
queryResult.Dataframes = tsdb.NewDecodedDataFrames(frames)
queryResult.Dataframes = plugins.NewDecodedDataFrames(frames)
return queryResult
}
@@ -214,7 +217,7 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin
plugin, ok := plugins.DataSources[dsInfo.Type]
plugin, ok := manager.DataSources[dsInfo.Type]
if !ok {
return nil, errors.New("unable to find datasource plugin Azure Monitor")
}
@@ -229,7 +232,8 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
return req, nil
}
func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (*plugins.AppPluginRoute, string, error) {
func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (
*plugins.AppPluginRoute, string, error) {
pluginRouteName := "loganalyticsazure"
switch cloudName {
@@ -240,7 +244,6 @@ func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourceP
}
var logAnalyticsRoute *plugins.AppPluginRoute
for _, route := range plugin.Routes {
if route.Path == pluginRouteName {
logAnalyticsRoute = route
@@ -11,7 +11,6 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/stretchr/testify/require"
)
@@ -21,18 +20,18 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
tests := []struct {
name string
queryModel []*tsdb.Query
timeRange *tsdb.TimeRange
queryModel []plugins.DataSubQuery
timeRange plugins.DataTimeRange
azureLogAnalyticsQueries []*AzureLogAnalyticsQuery
Err require.ErrorAssertionFunc
}{
{
name: "Query with macros should be interpolated",
timeRange: &tsdb.TimeRange{
timeRange: plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
queryModel: []*tsdb.Query{
queryModel: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
@@ -45,7 +44,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resultFormat": "time_series",
},
}),
RefId: "A",
RefID: "A",
},
},
azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{
@@ -17,12 +17,11 @@ import (
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/errutil"
opentracing "github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
"github.com/grafana/grafana/pkg/tsdb"
)
// AzureMonitorDatasource calls the Azure Monitor API - one of the four API's supported
@@ -42,25 +41,28 @@ const azureMonitorAPIVersion = "2018-01-01"
// 1. build the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format
func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []*tsdb.Query, timeRange *tsdb.TimeRange) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
for _, query := range queries {
queryRes, resp, err := e.executeQuery(ctx, query, originalQueries, timeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
err = e.parseResponse(queryRes, resp, query)
frames, err := e.parseResponse(resp, query)
if err != nil {
queryRes.Error = err
} else {
queryRes.Dataframes = frames
}
result.Results[query.RefID] = queryRes
}
@@ -68,7 +70,7 @@ func (e *AzureMonitorDatasource) executeTimeSeriesQuery(ctx context.Context, ori
return result, nil
}
func (e *AzureMonitorDatasource) buildQueries(queries []*tsdb.Query, timeRange *tsdb.TimeRange) ([]*AzureMonitorQuery, error) {
func (e *AzureMonitorDatasource) buildQueries(queries []plugins.DataSubQuery, timeRange plugins.DataTimeRange) ([]*AzureMonitorQuery, error) {
azureMonitorQueries := []*AzureMonitorQuery{}
startTime, err := timeRange.ParseFrom()
if err != nil {
@@ -115,7 +117,7 @@ func (e *AzureMonitorDatasource) buildQueries(queries []*tsdb.Query, timeRange *
timeGrain := azJSONModel.TimeGrain
timeGrains := azJSONModel.AllowedTimeGrainsMs
if timeGrain == "auto" {
timeGrain, err = setAutoTimeGrain(query.IntervalMs, timeGrains)
timeGrain, err = setAutoTimeGrain(query.IntervalMS, timeGrains)
if err != nil {
return nil, err
}
@@ -162,7 +164,7 @@ func (e *AzureMonitorDatasource) buildQueries(queries []*tsdb.Query, timeRange *
UrlComponents: urlComponents,
Target: target,
Params: params,
RefID: query.RefId,
RefID: query.RefID,
Alias: alias,
})
}
@@ -170,8 +172,9 @@ func (e *AzureMonitorDatasource) buildQueries(queries []*tsdb.Query, timeRange *
return azureMonitorQueries, nil
}
func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureMonitorQuery, queries []*tsdb.Query, timeRange *tsdb.TimeRange) (*tsdb.QueryResult, AzureMonitorResponse, error) {
queryResult := &tsdb.QueryResult{RefId: query.RefID}
func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureMonitorQuery, queries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) (plugins.DataQueryResult, AzureMonitorResponse, error) {
queryResult := plugins.DataQueryResult{RefID: query.RefID}
req, err := e.createRequest(ctx, e.dsInfo)
if err != nil {
@@ -223,7 +226,7 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *AzureM
func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) {
// find plugin
plugin, ok := plugins.DataSources[dsInfo.Type]
plugin, ok := manager.DataSources[dsInfo.Type]
if !ok {
return nil, errors.New("unable to find datasource plugin Azure Monitor")
}
@@ -280,9 +283,10 @@ func (e *AzureMonitorDatasource) unmarshalResponse(res *http.Response) (AzureMon
return data, nil
}
func (e *AzureMonitorDatasource) parseResponse(queryRes *tsdb.QueryResult, amr AzureMonitorResponse, query *AzureMonitorQuery) error {
func (e *AzureMonitorDatasource) parseResponse(amr AzureMonitorResponse, query *AzureMonitorQuery) (
plugins.DataFrames, error) {
if len(amr.Value) == 0 {
return nil
return nil, nil
}
frames := data.Frames{}
@@ -340,14 +344,13 @@ func (e *AzureMonitorDatasource) parseResponse(queryRes *tsdb.QueryResult, amr A
frames = append(frames, frame)
}
queryRes.Dataframes = tsdb.NewDecodedDataFrames(frames)
return nil
return plugins.NewDecodedDataFrames(frames), nil
}
// formatAzureMonitorLegendKey builds the legend key or timeseries name
// Alias patterns like {{resourcename}} are replaced with the appropriate data values.
func formatAzureMonitorLegendKey(alias string, resourceName string, metricName string, metadataName string, metadataValue string, namespace string, seriesID string, labels data.Labels) string {
func formatAzureMonitorLegendKey(alias string, resourceName string, metricName string, metadataName string,
metadataValue string, namespace string, seriesID string, labels data.Labels) string {
startIndex := strings.Index(seriesID, "/resourceGroups/") + 16
endIndex := strings.Index(seriesID, "/providers")
resourceGroup := seriesID[startIndex:endIndex]
@@ -14,7 +14,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require"
ptr "github.com/xorcare/pointer"
)
@@ -125,12 +125,12 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
for k, v := range commonAzureModelProps {
tt.azureMonitorVariedProperties[k] = v
}
tsdbQuery := &tsdb.TsdbQuery{
TimeRange: &tsdb.TimeRange{
tsdbQuery := plugins.DataQuery{
TimeRange: &plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
Queries: []*tsdb.Query{
Queries: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
@@ -142,8 +142,8 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
"azureMonitor": tt.azureMonitorVariedProperties,
},
),
RefId: "A",
IntervalMs: tt.queryIntervalMS,
RefID: "A",
IntervalMS: tt.queryIntervalMS,
},
},
}
@@ -161,7 +161,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) {
Alias: "testalias",
}
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
queries, err := datasource.buildQueries(tsdbQuery.Queries, *tsdbQuery.TimeRange)
require.NoError(t, err)
if diff := cmp.Diff(azureMonitorQuery, queries[0], cmpopts.IgnoreUnexported(simplejson.Json{}), cmpopts.IgnoreFields(AzureMonitorQuery{}, "Params")); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff)
@@ -430,15 +430,16 @@ func TestAzureMonitorParseResponse(t *testing.T) {
}
datasource := &AzureMonitorDatasource{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
azData := loadTestFile(t, "azuremonitor/"+tt.responseFile)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
err := datasource.parseResponse(res, azData, tt.mockQuery)
res := plugins.DataQueryResult{Meta: simplejson.New(), RefID: "A"}
require.NotNil(t, res)
dframes, err := datasource.parseResponse(azData, tt.mockQuery)
require.NoError(t, err)
require.NotNil(t, dframes)
frames, err := res.Dataframes.Decoded()
frames, err := dframes.Decoded()
require.NoError(t, err)
if diff := cmp.Diff(tt.expectedFrames, frames, data.FrameTestCompareOptions()...); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff)
+35 -24
View File
@@ -8,14 +8,30 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/registry"
)
var (
azlog log.Logger
legendKeyFormat *regexp.Regexp
azlog = log.New("tsdb.azuremonitor")
legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
)
func init() {
registry.Register(&registry.Descriptor{
Name: "AzureMonitorService",
InitPriority: registry.Low,
Instance: &Service{},
})
}
type Service struct {
}
func (s *Service) Init() error {
return nil
}
// AzureMonitorExecutor executes queries for the Azure Monitor datasource - all four services
type AzureMonitorExecutor struct {
httpClient *http.Client
@@ -23,7 +39,7 @@ type AzureMonitorExecutor struct {
}
// NewAzureMonitorExecutor initializes a http client
func NewAzureMonitorExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
httpClient, err := dsInfo.GetHttpClient()
if err != nil {
return nil, err
@@ -35,23 +51,18 @@ func NewAzureMonitorExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint,
}, nil
}
func init() {
azlog = log.New("tsdb.azuremonitor")
tsdb.RegisterTsdbQueryEndpoint("grafana-azure-monitor-datasource", NewAzureMonitorExecutor)
legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
}
// Query takes in the frontend queries, parses them into the query format
// expected by chosen Azure Monitor service (Azure Monitor, App Insights etc.)
// executes the queries against the API and parses the response into
// the right format
func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
tsdbQuery plugins.DataQuery) (plugins.DataResponse, error) {
var err error
var azureMonitorQueries []*tsdb.Query
var applicationInsightsQueries []*tsdb.Query
var azureLogAnalyticsQueries []*tsdb.Query
var insightsAnalyticsQueries []*tsdb.Query
var azureMonitorQueries []plugins.DataSubQuery
var applicationInsightsQueries []plugins.DataSubQuery
var azureLogAnalyticsQueries []plugins.DataSubQuery
var insightsAnalyticsQueries []plugins.DataSubQuery
for _, query := range tsdbQuery.Queries {
queryType := query.Model.Get("queryType").MustString("")
@@ -66,7 +77,7 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
case "Insights Analytics":
insightsAnalyticsQueries = append(insightsAnalyticsQueries, query)
default:
return nil, fmt.Errorf("alerting not supported for %q", queryType)
return plugins.DataResponse{}, fmt.Errorf("alerting not supported for %q", queryType)
}
}
@@ -90,24 +101,24 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
dsInfo: e.dsInfo,
}
azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, tsdbQuery.TimeRange)
azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, *tsdbQuery.TimeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
aiResult, err := aiDatasource.executeTimeSeriesQuery(ctx, applicationInsightsQueries, tsdbQuery.TimeRange)
aiResult, err := aiDatasource.executeTimeSeriesQuery(ctx, applicationInsightsQueries, *tsdbQuery.TimeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
alaResult, err := alaDatasource.executeTimeSeriesQuery(ctx, azureLogAnalyticsQueries, tsdbQuery.TimeRange)
alaResult, err := alaDatasource.executeTimeSeriesQuery(ctx, azureLogAnalyticsQueries, *tsdbQuery.TimeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
iaResult, err := iaDatasource.executeTimeSeriesQuery(ctx, insightsAnalyticsQueries, tsdbQuery.TimeRange)
iaResult, err := iaDatasource.executeTimeSeriesQuery(ctx, insightsAnalyticsQueries, *tsdbQuery.TimeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
for k, v := range aiResult.Results {
@@ -15,8 +15,8 @@ import (
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
@@ -39,14 +39,15 @@ type InsightsAnalyticsQuery struct {
Target string
}
func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []*tsdb.Query, timeRange *tsdb.TimeRange) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context,
originalQueries []plugins.DataSubQuery, timeRange plugins.DataTimeRange) (plugins.DataResponse, error) {
result := plugins.DataResponse{
Results: map[string]plugins.DataQueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return nil, err
return plugins.DataResponse{}, err
}
for _, query := range queries {
@@ -56,7 +57,8 @@ func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context
return result, nil
}
func (e *InsightsAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRange *tsdb.TimeRange) ([]*InsightsAnalyticsQuery, error) {
func (e *InsightsAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) ([]*InsightsAnalyticsQuery, error) {
iaQueries := []*InsightsAnalyticsQuery{}
for _, query := range queries {
@@ -74,7 +76,7 @@ func (e *InsightsAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRa
qm.RawQuery = queryJSONModel.InsightsAnalytics.Query
qm.ResultFormat = queryJSONModel.InsightsAnalytics.ResultFormat
qm.RefID = query.RefId
qm.RefID = query.RefID
if qm.RawQuery == "" {
return nil, fmt.Errorf("query is missing query string property")
@@ -94,10 +96,10 @@ func (e *InsightsAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRa
return iaQueries, nil
}
func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery) *tsdb.QueryResult {
queryResult := &tsdb.QueryResult{RefId: query.RefID}
func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery) plugins.DataQueryResult {
queryResult := plugins.DataQueryResult{RefID: query.RefID}
queryResultError := func(err error) *tsdb.QueryResult {
queryResultError := func(err error) plugins.DataQueryResult {
queryResult.Error = err
return queryResult
}
@@ -170,19 +172,22 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I
if err == nil {
frame = wideFrame
} else {
frame.AppendNotices(data.Notice{Severity: data.NoticeSeverityWarning, Text: "could not convert frame to time series, returning raw table: " + err.Error()})
frame.AppendNotices(data.Notice{
Severity: data.NoticeSeverityWarning,
Text: "could not convert frame to time series, returning raw table: " + err.Error(),
})
}
}
}
frames := data.Frames{frame}
queryResult.Dataframes = tsdb.NewDecodedDataFrames(frames)
queryResult.Dataframes = plugins.NewDecodedDataFrames(frames)
return queryResult
}
func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) {
// find plugin
plugin, ok := plugins.DataSources[dsInfo.Type]
plugin, ok := manager.DataSources[dsInfo.Type]
if !ok {
return nil, errors.New("unable to find datasource plugin Azure Application Insights")
}
@@ -215,7 +220,8 @@ func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
return req, nil
}
func (e *InsightsAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (*plugins.AppPluginRoute, string, error) {
func (e *InsightsAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (
*plugins.AppPluginRoute, string, error) {
pluginRouteName := "appinsights"
if cloudName == "chinaazuremonitor" {
@@ -223,7 +229,6 @@ func (e *InsightsAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourceP
}
var pluginRoute *plugins.AppPluginRoute
for _, route := range plugin.Routes {
if route.Path == pluginRouteName {
pluginRoute = route
+15 -12
View File
@@ -6,7 +6,8 @@ import (
"strings"
"time"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
const rsIdentifier = `__(timeFilter|timeFrom|timeTo|interval|contains|escapeMulti)`
@@ -14,8 +15,8 @@ const sExpr = `\$` + rsIdentifier + `(?:\(([^\)]*)\))?`
const escapeMultiExpr = `\$__escapeMulti\(('.*')\)`
type kqlMacroEngine struct {
timeRange *tsdb.TimeRange
query *tsdb.Query
timeRange plugins.DataTimeRange
query plugins.DataSubQuery
}
// Macros:
@@ -28,7 +29,7 @@ type kqlMacroEngine struct {
// - $__escapeMulti('\\vm\eth0\Total','\\vm\eth2\Total') -> @'\\vm\eth0\Total',@'\\vm\eth2\Total'
// KqlInterpolate interpolates macros for Kusto Query Language (KQL) queries
func KqlInterpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, kql string, defaultTimeField ...string) (string, error) {
func KqlInterpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, kql string, defaultTimeField ...string) (string, error) {
engine := kqlMacroEngine{}
defaultTimeFieldForAllDatasources := "timestamp"
@@ -38,7 +39,7 @@ func KqlInterpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, kql string, de
return engine.Interpolate(query, timeRange, kql, defaultTimeFieldForAllDatasources)
}
func (m *kqlMacroEngine) Interpolate(query *tsdb.Query, timeRange *tsdb.TimeRange, kql string, defaultTimeField string) (string, error) {
func (m *kqlMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, kql string, defaultTimeField string) (string, error) {
m.timeRange = timeRange
m.query = query
rExp, _ := regexp.Compile(sExpr)
@@ -90,28 +91,30 @@ func (m *kqlMacroEngine) evaluateMacro(name string, defaultTimeField string, arg
if len(args) > 0 && args[0] != "" {
timeColumn = args[0]
}
return fmt.Sprintf("['%s'] >= datetime('%s') and ['%s'] <= datetime('%s')", timeColumn, m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339), timeColumn, m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
return fmt.Sprintf("['%s'] >= datetime('%s') and ['%s'] <= datetime('%s')", timeColumn,
m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339), timeColumn,
m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
case "timeFrom", "__from":
return fmt.Sprintf("datetime('%s')", m.timeRange.GetFromAsTimeUTC().Format(time.RFC3339)), nil
case "timeTo", "__to":
return fmt.Sprintf("datetime('%s')", m.timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
case "interval":
var interval time.Duration
if m.query.IntervalMs == 0 {
var it time.Duration
if m.query.IntervalMS == 0 {
to := m.timeRange.MustGetTo().UnixNano()
from := m.timeRange.MustGetFrom().UnixNano()
// default to "100 datapoints" if nothing in the query is more specific
defaultInterval := time.Duration((to - from) / 60)
var err error
interval, err = tsdb.GetIntervalFrom(m.query.DataSource, m.query.Model, defaultInterval)
it, err = interval.GetIntervalFrom(m.query.DataSource, m.query.Model, defaultInterval)
if err != nil {
azlog.Warn("Unable to get interval from query", "datasource", m.query.DataSource, "model", m.query.Model)
interval = defaultInterval
it = defaultInterval
}
} else {
interval = time.Millisecond * time.Duration(m.query.IntervalMs)
it = time.Millisecond * time.Duration(m.query.IntervalMS)
}
return fmt.Sprintf("%dms", int(interval/time.Millisecond)), nil
return fmt.Sprintf("%dms", int(it/time.Millisecond)), nil
case "contains":
if len(args) < 2 || args[0] == "" || args[1] == "" {
return "", fmt.Errorf("macro %v needs colName and variableSet", name)
+17 -17
View File
@@ -9,77 +9,77 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require"
)
func TestAzureLogAnalyticsMacros(t *testing.T) {
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
timeRange := &tsdb.TimeRange{
timeRange := plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
}
tests := []struct {
name string
query *tsdb.Query
timeRange *tsdb.TimeRange
query plugins.DataSubQuery
timeRange plugins.DataTimeRange
kql string
expected string
Err require.ErrorAssertionFunc
}{
{
name: "invalid macro should be ignored",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "$__invalid()",
expected: "$__invalid()",
Err: require.NoError,
},
{
name: "Kusto variables should be ignored",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: ") on $left.b == $right.y",
expected: ") on $left.b == $right.y",
Err: require.NoError,
},
{
name: "$__contains macro with a multi template variable that has multiple selected values as a parameter should build in clause",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "$__contains(col, 'val1','val2')",
expected: "['col'] in ('val1','val2')",
Err: require.NoError,
},
{
name: "$__contains macro with a multi template variable that has a single selected value as a parameter should build in clause",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "$__contains(col, 'val1' )",
expected: "['col'] in ('val1')",
Err: require.NoError,
},
{
name: "$__contains macro with multi template variable has custom All value as a parameter should return a true expression",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "$__contains(col, all)",
expected: "1 == 1",
Err: require.NoError,
},
{
name: "$__timeFilter has no column parameter should use default time field",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "$__timeFilter()",
expected: "['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z')",
Err: require.NoError,
},
{
name: "$__timeFilter has time field parameter",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "$__timeFilter(myTimeField)",
expected: "['myTimeField'] >= datetime('2018-03-15T13:00:00Z') and ['myTimeField'] <= datetime('2018-03-15T13:34:00Z')",
Err: require.NoError,
},
{
name: "$__timeFrom and $__timeTo is in the query and range is a specific interval",
query: &tsdb.Query{},
query: plugins.DataSubQuery{},
kql: "myTimeField >= $__timeFrom() and myTimeField <= $__timeTo()",
expected: "myTimeField >= datetime('2018-03-15T13:00:00Z') and myTimeField <= datetime('2018-03-15T13:34:00Z')",
Err: require.NoError,
@@ -87,7 +87,7 @@ func TestAzureLogAnalyticsMacros(t *testing.T) {
{
name: "$__interval should use the defined interval from the query",
timeRange: timeRange,
query: &tsdb.Query{
query: plugins.DataSubQuery{
Model: simplejson.NewFromAny(map[string]interface{}{
"interval": "5m",
}),
@@ -98,7 +98,7 @@ func TestAzureLogAnalyticsMacros(t *testing.T) {
},
{
name: "$__interval should use the default interval if none is specified",
query: &tsdb.Query{
query: plugins.DataSubQuery{
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
@@ -108,7 +108,7 @@ func TestAzureLogAnalyticsMacros(t *testing.T) {
},
{
name: "$__escapeMulti with multi template variable should replace values with KQL style escaped strings",
query: &tsdb.Query{
query: plugins.DataSubQuery{
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
@@ -118,7 +118,7 @@ func TestAzureLogAnalyticsMacros(t *testing.T) {
},
{
name: "$__escapeMulti with multi template variable and has one selected value that contains comma",
query: &tsdb.Query{
query: plugins.DataSubQuery{
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
@@ -128,7 +128,7 @@ func TestAzureLogAnalyticsMacros(t *testing.T) {
},
{
name: "$__escapeMulti with multi template variable and is not wrapped in single quotes should fail",
query: &tsdb.Query{
query: plugins.DataSubQuery{
DataSource: &models.DataSource{},
Model: simplejson.NewFromAny(map[string]interface{}{}),
},
+4 -4
View File
@@ -6,7 +6,7 @@ import (
"strings"
"time"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
// TimeGrain handles conversions between
@@ -18,8 +18,8 @@ var (
smallTimeUnits = []string{"hour", "minute", "h", "m"}
)
func (tg *TimeGrain) createISO8601DurationFromIntervalMS(interval int64) (string, error) {
formatted := tsdb.FormatDuration(time.Duration(interval) * time.Millisecond)
func (tg *TimeGrain) createISO8601DurationFromIntervalMS(it int64) (string, error) {
formatted := interval.FormatDuration(time.Duration(it) * time.Millisecond)
if strings.Contains(formatted, "ms") {
return "PT1M", nil
@@ -28,7 +28,7 @@ func (tg *TimeGrain) createISO8601DurationFromIntervalMS(interval int64) (string
timeValueString := formatted[0 : len(formatted)-1]
timeValue, err := strconv.Atoi(timeValueString)
if err != nil {
return "", fmt.Errorf("could not parse interval %q to an ISO 8061 duration: %w", interval, err)
return "", fmt.Errorf("could not parse interval %q to an ISO 8061 duration: %w", it, err)
}
unit := formatted[len(formatted)-1:]