diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 47c88b9c3b9..00e5d96765e 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -4088,14 +4088,9 @@ "count": 3 } }, - "public/app/plugins/datasource/graphite/datasource.test.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 3 - } - }, "public/app/plugins/datasource/graphite/datasource.ts": { "@typescript-eslint/consistent-type-assertions": { - "count": 4 + "count": 3 }, "@typescript-eslint/no-explicit-any": { "count": 8 diff --git a/pkg/tsdb/graphite/query.go b/pkg/tsdb/graphite/query.go index 331dbc1af34..460ac2cfba5 100644 --- a/pkg/tsdb/graphite/query.go +++ b/pkg/tsdb/graphite/query.go @@ -123,11 +123,11 @@ func (s *Service) RunQuery(ctx context.Context, req *backend.QueryDataRequest, d // processQuery converts a Graphite data source query to a Graphite query target. It returns the target, // and the model if the target is invalid -func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, error) { +func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, bool, error) { queryJSON := GraphiteQuery{} err := json.Unmarshal(query.JSON, &queryJSON) if err != nil { - return "", &queryJSON, fmt.Errorf("failed to decode the Graphite query: %w", err) + return "", &queryJSON, false, fmt.Errorf("failed to decode the Graphite query: %w", err) } s.logger.Debug("Graphite", "query", queryJSON) currTarget := queryJSON.TargetFull @@ -137,11 +137,11 @@ func (s *Service) processQuery(query backend.DataQuery) (string, *GraphiteQuery, } if currTarget == "" { s.logger.Debug("Graphite", "empty query target", queryJSON) - return "", &queryJSON, nil + return "", &queryJSON, false, nil } target := fixIntervalFormat(currTarget) - return target, nil, nil + return target, nil, queryJSON.IsMetricTank, nil } func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQuery, dsInfo *datasourceInfo) (*http.Request, url.Values, *GraphiteQuery, error) { @@ -158,7 +158,7 @@ func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQ "target": []string{}, } - target, emptyQuery, err := s.processQuery(query) + target, emptyQuery, isMetricTank, err := s.processQuery(query) if err != nil { return nil, formData, nil, err } @@ -172,11 +172,17 @@ func (s *Service) createGraphiteRequest(ctx context.Context, query backend.DataQ s.logger.Debug("Graphite request", "params", formData) + params := map[string][]string{} + if isMetricTank { + params["meta"] = []string{"true"} + } + graphiteReq, err := s.createRequest(ctx, dsInfo, URLParams{ - SubPath: "render", - Method: http.MethodPost, - Body: strings.NewReader(formData.Encode()), - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SubPath: "render", + Method: http.MethodPost, + Body: strings.NewReader(formData.Encode()), + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + QueryParams: params, }) if err != nil { return nil, formData, nil, err diff --git a/pkg/tsdb/graphite/query_test.go b/pkg/tsdb/graphite/query_test.go index b49479a7c8a..c7434099508 100644 --- a/pkg/tsdb/graphite/query_test.go +++ b/pkg/tsdb/graphite/query_test.go @@ -32,10 +32,11 @@ func TestProcessQuery(t *testing.T) { }`), }, } - target, jsonModel, err := service.processQuery(queries[0]) + target, jsonModel, isMetricTank, err := service.processQuery(queries[0]) assert.NoError(t, err) assert.Nil(t, jsonModel) assert.Equal(t, "app.grafana.*.dashboards.views.1M.count", target) + assert.False(t, isMetricTank) }) t.Run("Returns if target is empty", func(t *testing.T) { @@ -48,10 +49,26 @@ func TestProcessQuery(t *testing.T) { }, } emptyQuery := GraphiteQuery{Target: ""} - target, jsonModel, err := service.processQuery(queries[0]) + target, jsonModel, isMetricTank, err := service.processQuery(queries[0]) assert.NoError(t, err) assert.Equal(t, &emptyQuery, jsonModel) assert.Equal(t, "", target) + assert.False(t, isMetricTank) + }) + + t.Run("Returns isMetricTank value", func(t *testing.T) { + queries := []backend.DataQuery{ + { + RefID: "A", + JSON: []byte(`{ + "target": "app.grafana.*.dashboards.views.1M.count", + "isMetricTank": true + }`), + }, + } + _, _, isMetricTank, err := service.processQuery(queries[0]) + assert.NoError(t, err) + assert.True(t, isMetricTank) }) t.Run("QueryData with no valid queries returns bad request response", func(t *testing.T) { diff --git a/pkg/tsdb/graphite/types.go b/pkg/tsdb/graphite/types.go index 04a3c8e812a..7e04a0d6e87 100644 --- a/pkg/tsdb/graphite/types.go +++ b/pkg/tsdb/graphite/types.go @@ -27,6 +27,7 @@ type GraphiteQuery struct { TargetFull string `json:"targetFull,omitempty"` Tags []string `json:"tags,omitempty"` FromAnnotations *bool `json:"fromAnnotations,omitempty"` + IsMetricTank bool `json:"isMetricTank,omitempty"` } type GraphiteEventsRequest struct { diff --git a/public/app/plugins/datasource/graphite/datasource.test.ts b/public/app/plugins/datasource/graphite/datasource.test.ts index 0f892638eec..fb0e90e8271 100644 --- a/public/app/plugins/datasource/graphite/datasource.test.ts +++ b/public/app/plugins/datasource/graphite/datasource.test.ts @@ -1,25 +1,37 @@ import { isArray } from 'lodash'; import moment from 'moment'; -import { of } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { AbstractLabelMatcher, AbstractLabelOperator, + CoreApp, DataQueryRequest, + DataQueryResponse, dateMath, dateTime, getFrameDisplayName, MetricFindValue, + PluginType, ScopedVars, } from '@grafana/data'; -import { BackendSrvRequest, FetchResponse, getTemplateSrv, TemplateSrv, VariableInterpolation } from '@grafana/runtime'; +import { + BackendSrvRequest, + config, + FetchResponse, + getTemplateSrv, + TemplateSrv, + VariableInterpolation, +} from '@grafana/runtime'; import { fromString } from './configuration/parseLokiLabelMappings'; import { GraphiteDatasource } from './datasource'; -import { GraphiteQuery, GraphiteQueryType } from './types'; +import { GraphiteQuery, GraphiteQueryType, GraphiteType } from './types'; import { DEFAULT_GRAPHITE_VERSION } from './versions'; const fetchMock = jest.fn(); +const postResourceMock = jest.fn(); +const getResourceMock = jest.fn(); jest.mock('@grafana/runtime', () => ({ ...(jest.requireActual('@grafana/runtime') as unknown as object), @@ -50,21 +62,53 @@ const createFetchResponse = (data: T): FetchResponse => ({ ok: true, }); +const instanceSettings = { + id: 1, + uid: 'graphiteUid', + type: 'graphite', + readOnly: false, + access: 'proxy' as 'proxy', + meta: { + id: '1', + name: 'graphite', + type: PluginType.datasource, + info: { + description: 'Graphite datasource', + author: { + name: 'Grafana Labs', + url: 'https://grafana.com', + }, + keywords: ['graphite', 'datasource'], + links: [], + logos: { + large: '', + small: '', + }, + screenshots: [], + updated: '', + version: '', + }, + module: '', + baseUrl: '', + }, + url: '/api/datasources/proxy/1', + name: 'graphiteProd', + jsonData: { + rollupIndicatorEnabled: true, + graphiteType: GraphiteType.Default, + }, +}; + describe('graphiteDatasource', () => { let ctx = {} as Context; beforeEach(() => { jest.clearAllMocks(); - const instanceSettings = { - url: '/api/datasources/proxy/1', - name: 'graphiteProd', - jsonData: { - rollupIndicatorEnabled: true, - }, - }; const templateSrv = getTemplateSrv(); const ds = new GraphiteDatasource(instanceSettings, templateSrv); + ds.postResource = postResourceMock; + ds.getResource = getResourceMock; ctx = { templateSrv, ds }; }); @@ -186,19 +230,33 @@ describe('graphiteDatasource', () => { }); describe('When querying graphite with one target using query editor target spec', () => { - let response: unknown; + let response: Observable; let requestOptions: BackendSrvRequest; beforeEach(() => { - const query = { + const query: DataQueryRequest = { panelId: 3, - dashboardId: 5, - range: { from: dateTime('2022-04-01T00:00:00'), to: dateTime('2022-07-01T00:00:00') }, + dashboardUID: '5', + range: { + from: dateTime('2022-04-01T00:00:00'), + to: dateTime('2022-07-01T00:00:00'), + raw: { + from: '1648789200', + to: '1656655200', + }, + }, targets: [ { target: 'prod1.count', refId: 'A' }, { target: 'prod2.count', refId: 'B' }, ], maxDataPoints: 500, + requestId: '', + interval: '1m', + intervalMs: 60000, + scopedVars: {}, + timezone: 'utc', + app: CoreApp.Dashboard, + startTime: 0, }; fetchMock.mockImplementation((options) => { requestOptions = options; @@ -215,11 +273,11 @@ describe('graphiteDatasource', () => { ); }); - response = ctx.ds.query(query as unknown as DataQueryRequest); + response = ctx.ds.query(query); }); it('X-Dashboard and X-Panel headers to be set!', () => { - expect(requestOptions.headers?.['X-Dashboard-Id']).toBe(5); + expect(requestOptions.headers?.['X-Dashboard-Id']).toBe('5'); expect(requestOptions.headers?.['X-Panel-Id']).toBe(3); }); @@ -245,7 +303,7 @@ describe('graphiteDatasource', () => { }); it('should return series list', async () => { - await expect(response).toEmitValuesWith((values: any) => { + await expect(response).toEmitValuesWith((values: DataQueryResponse[]) => { const results = values[0]; expect(results.data.length).toBe(1); expect(results.data[0].name).toBe('prod1.count'); @@ -253,7 +311,7 @@ describe('graphiteDatasource', () => { }); it('should convert to millisecond resolution', async () => { - await expect(response).toEmitValuesWith((values: any) => { + await expect(response).toEmitValuesWith((values: DataQueryResponse[]) => { const results = values[0]; expect(results.data[0].fields[1].values[0]).toBe(10); }); @@ -261,7 +319,12 @@ describe('graphiteDatasource', () => { }); describe('when fetching Graphite Events as annotations', () => { - let results: any; + let results: Array<{ + annotation: string; + time: string; + title: string; + tags?: string[]; + }>; let errorSpy: jest.SpyInstance; beforeEach(() => { @@ -278,7 +341,8 @@ describe('graphiteDatasource', () => { fromAnnotations: true, tags: ['tag1'], queryType: 'tags', - } as GraphiteQuery, + refId: 'Anno', + }, ], range: { @@ -313,6 +377,9 @@ describe('graphiteDatasource', () => { it('should parse the tags string into an array', () => { expect(isArray(results[0].tags)).toEqual(true); + if (!results[0].tags) { + throw new Error('tags not defined'); + } expect(results[0].tags.length).toEqual(2); expect(results[0].tags[0]).toEqual('tag1'); expect(results[0].tags[1]).toEqual('tag2'); @@ -342,6 +409,9 @@ describe('graphiteDatasource', () => { it('should parse the tags string into an array', () => { expect(isArray(results[0].tags)).toEqual(true); + if (!results[0].tags) { + throw new Error('tags not defined'); + } expect(results[0].tags.length).toEqual(2); expect(results[0].tags[0]).toEqual('tag1'); expect(results[0].tags[1]).toEqual('tag2'); @@ -360,6 +430,120 @@ describe('graphiteDatasource', () => { }); }); + describe('when fetching Graphite Events as annotations (backend)', () => { + let results: Array<{ + annotation: string; + time: string; + title: string; + tags?: string[]; + }>; + let errorSpy: jest.SpyInstance; + + beforeEach(() => { + errorSpy = jest.spyOn(console, 'error').mockImplementation(); + config.featureToggles.graphiteBackendMode = true; + }); + + afterEach(() => { + errorSpy.mockRestore(); + config.featureToggles.graphiteBackendMode = false; + }); + + const options = { + targets: [ + { + fromAnnotations: true, + tags: ['tag1'], + queryType: 'tags', + refId: 'Anno', + }, + ], + + range: { + from: dateTime('2022-06-06T07:03:03.109Z'), + to: dateTime('2022-06-07T07:03:03.109Z'), + raw: { + from: '2022-06-06T07:03:03.109Z', + to: '2022-06-07T07:03:03.109Z', + }, + }, + }; + + describe('and tags are returned as string', () => { + const response = [ + { + when: 1507222850, + tags: 'tag1 tag2', + data: 'some text', + id: 2, + what: 'Event - deploy', + }, + ]; + + beforeEach(async () => { + postResourceMock.mockImplementation((options) => { + return createFetchResponse(response); + }); + await ctx.ds.annotationEvents(options.range, options.targets[0]).then((data) => { + results = data; + }); + }); + + it('should parse the tags string into an array', () => { + expect(isArray(results[0].tags)).toEqual(true); + if (!results[0].tags) { + throw new Error('tags not defined'); + } + expect(results[0].tags.length).toEqual(2); + expect(results[0].tags[0]).toEqual('tag1'); + expect(results[0].tags[1]).toEqual('tag2'); + }); + }); + + describe('and tags are returned as an array', () => { + const response = [ + { + when: 1507222850, + tags: ['tag1', 'tag2'], + data: 'some text', + id: 2, + what: 'Event - deploy', + }, + ]; + + beforeEach(async () => { + fetchMock.mockImplementation((options) => { + return of(createFetchResponse(response)); + }); + + await ctx.ds.annotationEvents(options.range, options.targets[0]).then((data) => { + results = data; + }); + }); + + it('should parse the tags string into an array', () => { + expect(isArray(results[0].tags)).toEqual(true); + if (!results[0].tags) { + throw new Error('tags not defined'); + } + expect(results[0].tags.length).toEqual(2); + expect(results[0].tags[0]).toEqual('tag1'); + expect(results[0].tags[1]).toEqual('tag2'); + }); + }); + + it('and tags response is invalid', async () => { + postResourceMock.mockImplementation((options) => { + return createFetchResponse('zzzzzzz'); + }); + await ctx.ds.annotationEvents(options.range, options.targets[0]).then((data) => { + results = data; + }); + expect(results).toEqual([]); + expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/Unable to get annotations/)); + }); + }); + describe('when fetching Graphite function descriptions', () => { // `"default": Infinity` (invalid JSON) in params passed by Graphite API in 1.1.7 const INVALID_JSON = @@ -392,11 +576,17 @@ describe('graphiteDatasource', () => { }); describe('building graphite params', () => { + const defaultQueryProperties = { + from: 'now-1h', + until: 'now', + format: 'json', + }; it('should return empty array if no targets', () => { const originalTargetMap = { A: '' }; const results = ctx.ds.buildGraphiteParams( { - targets: [{}], + targets: [], + ...defaultQueryProperties, }, originalTargetMap ); @@ -410,7 +600,11 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'prod1.{test,test2}' }, { target: 'prod2.count' }], + targets: [ + { target: 'prod1.{test,test2}', refId: 'A' }, + { target: 'prod2.count', refId: 'B' }, + ], + ...defaultQueryProperties, }, originalTargetMap ); @@ -425,7 +619,12 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'series1' }, { target: 'series2' }, { target: 'asPercent(#A,#B)' }], + targets: [ + { target: 'series1', refId: 'A' }, + { target: 'series2', refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C ' }, + ], + ...defaultQueryProperties, }, originalTargetMap ); @@ -441,10 +640,11 @@ describe('graphiteDatasource', () => { const results = ctx.ds.buildGraphiteParams( { targets: [ - { target: 'series1', hide: true }, - { target: 'sumSeries(#A)', hide: true }, - { target: 'asPercent(#A,#B)' }, + { target: 'series1', hide: true, refId: 'A' }, + { target: 'sumSeries(#A)', hide: true, refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, ], + ...defaultQueryProperties, }, originalTargetMap ); @@ -459,7 +659,12 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'series1' }, { target: 'sumSeries(#A)' }, { target: 'asPercent(#A,#B)' }], + targets: [ + { target: 'series1', refId: 'A' }, + { target: 'sumSeries(#A)', refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, + ], + ...defaultQueryProperties, }, originalTargetMap ); @@ -490,7 +695,12 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: '[[metric]]' }, { target: 'sumSeries(#A)' }, { target: 'asPercent(#A,#B)' }], + targets: [ + { target: '[[metric]]', refId: 'A' }, + { target: 'sumSeries(#A)', refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, + ], + ...defaultQueryProperties, }, originalTargetMap ); @@ -526,7 +736,11 @@ describe('graphiteDatasource', () => { const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: '$metric' }, { target: 'sumSeries(#A)' }], + targets: [ + { target: '$metric', refId: 'A' }, + { target: 'sumSeries(#A)', refId: 'B' }, + ], + ...defaultQueryProperties, }, originalTargetMap, scopedVars @@ -564,7 +778,11 @@ describe('graphiteDatasource', () => { const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: '$server.cpu', hide: true }, { target: 'avg(#A)' }], + targets: [ + { target: '$server.cpu', hide: true, refId: 'A' }, + { target: 'avg(#A)', refId: 'B' }, + ], + ...defaultQueryProperties, }, originalTargetMap, scopedVars @@ -579,7 +797,8 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'sumSeries(carbon.test.test-host.cpuUsage, #A)' }], + targets: [{ target: 'sumSeries(carbon.test.test-host.cpuUsage, #A)', refId: 'A' }], + ...defaultQueryProperties, }, originalTargetMap ); @@ -599,11 +818,14 @@ describe('graphiteDatasource', () => { targets: [ { target: 'sumSeries(carbon.test.test-host.cpuUsage, #A, #B)', + refId: 'A', }, { target: 'add(carbon.test.test-host.cpuUsage, 1.5)', + refId: 'B', }, ], + ...defaultQueryProperties, }, originalTargetMap ); @@ -621,7 +843,8 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: "summarize(prod.25m.count, '25m', 'sum')" }], + targets: [{ target: "summarize(prod.25m.count, '25m', 'sum')", refId: 'A' }], + ...defaultQueryProperties, }, originalTargetMap ); @@ -634,7 +857,8 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: "summarize(prod.5M.count, '5M', 'sum')" }], + targets: [{ target: "summarize(prod.5M.count, '5M', 'sum')", refId: 'A' }], + ...defaultQueryProperties, }, originalTargetMap ); @@ -648,11 +872,15 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'series1' }, { target: '' }], + targets: [ + { target: 'series1', refId: 'A' }, + { target: '', refId: 'B' }, + ], + ...defaultQueryProperties, }, originalTargetMap ); - expect(results.length).toBe(2); + expect(results.length).toBe(4); }); describe('when formatting targets', () => { @@ -679,11 +907,12 @@ describe('graphiteDatasource', () => { }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'my.$metric.*' }], + targets: [{ target: 'my.$metric.*', refId: 'A' }], + ...defaultQueryProperties, }, originalTargetMap ); - expect(results).toStrictEqual(['target=my.b.*', 'format=json']); + expect(results).toStrictEqual(['target=my.b.*', 'from=now-1h', 'until=now', 'format=json']); }); it('globs for more than one variable', () => { @@ -707,12 +936,345 @@ describe('graphiteDatasource', () => { const originalTargetMap = { A: 'my.[[metric]].*' }; const results = ctx.ds.buildGraphiteParams( { - targets: [{ target: 'my.[[metric]].*' }], + targets: [{ target: 'my.[[metric]].*', refId: 'A' }], + ...defaultQueryProperties, }, originalTargetMap ); - expect(results).toStrictEqual(['target=my.%7Ba%2Cb%7D.*', 'format=json']); + expect(results).toEqual(['target=my.%7Ba%2Cb%7D.*', 'from=now-1h', 'until=now', 'format=json']); + }); + }); + }); + + describe('building graphite queries (backend)', () => { + const defaultQueryProperties = { + requestId: 'reqId', + interval: '1s', + intervalMs: 1000, + maxDataPoints: 100, + range: { + from: dateTime('2022-06-06T07:03:03.109Z'), + to: dateTime('2022-06-07T07:03:03.109Z'), + raw: { + from: '2022-06-06T07:03:03.109Z', + to: '2022-06-07T07:03:03.109Z', + }, + }, + scopedVars: {}, + timezone: '', + app: 'dashboards', + startTime: 0, + }; + it('should return empty array if no targets', () => { + const originalTargetMap = { A: '' }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + targets: [], + ...defaultQueryProperties, + }, + originalTargetMap + ); + expect(results.length).toBe(0); + }); + + it('should replace target placeholder', () => { + const originalTargetMap = { + A: 'series1', + B: 'series2', + C: 'asPercent(#A,#B)', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: 'series1', refId: 'A' }, + { target: 'series2', refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, + ], + }, + originalTargetMap + ); + expect(results[2].target).toBe('asPercent(series1,series2)'); + }); + + it('should replace target placeholder for hidden series', () => { + const originalTargetMap = { + A: 'series1', + B: 'sumSeries(#A)', + C: 'asPercent(#A,#B)', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: 'series1', hide: true, refId: 'A' }, + { target: 'sumSeries(#A)', hide: true, refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, + ], + }, + originalTargetMap + ); + expect(results[0].target).toBe('asPercent(series1,sumSeries(series1))'); + }); + + it('should replace target placeholder when nesting query references', () => { + const originalTargetMap = { + A: 'series1', + B: 'sumSeries(#A)', + C: 'asPercent(#A,#B)', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: 'series1', refId: 'A' }, + { target: 'sumSeries(#A)', refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, + ], + }, + originalTargetMap + ); + expect(results[2].target).toBe('asPercent(series1,sumSeries(series1))'); + }); + + it('should replace target placeholder when nesting query references with template variables', () => { + const originalReplace = ctx.templateSrv.replace; + ctx.templateSrv.replace = jest + .fn() + .mockImplementation( + ( + target?: string | undefined, + scopedVars?: ScopedVars | undefined, + format?: string | Function, + interpolations?: VariableInterpolation[] + ): string => { + if (target?.includes('[[metric]]')) { + return target.replaceAll('[[metric]]', 'aMetricName'); + } + return originalReplace(target, scopedVars, format, interpolations); + } + ); + const originalTargetMap = { + A: '[[metric]]', + B: 'sumSeries(#A)', + C: 'asPercent(#A,#B)', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: '[[metric]]', refId: 'A' }, + { target: 'sumSeries(#A)', refId: 'B' }, + { target: 'asPercent(#A,#B)', refId: 'C' }, + ], + }, + originalTargetMap + ); + expect(results[2].target).toBe('asPercent(aMetricName,sumSeries(aMetricName))'); + }); + + it('should use scoped variables when nesting query references', () => { + const originalReplace = ctx.templateSrv.replace; + ctx.templateSrv.replace = jest + .fn() + .mockImplementation( + ( + target?: string | undefined, + scopedVars?: ScopedVars | undefined, + format?: string | Function, + interpolations?: VariableInterpolation[] + ): string => { + if (target?.includes('$metric')) { + return target.replaceAll('$metric', 'scopedValue'); + } + return originalReplace(target, scopedVars, format, interpolations); + } + ); + + const originalTargetMap = { + A: '$metric', + B: 'sumSeries(#A)', + }; + + const scopedVars = { + metric: { text: 'scopedValue', value: 'scopedValue' }, + }; + + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: '$metric', refId: 'A' }, + { target: 'sumSeries(#A)', refId: 'B' }, + ], + scopedVars, + }, + originalTargetMap + ); + + expect(results[1].target).toBe('sumSeries(scopedValue)'); + }); + + it('should apply scoped variables to nested references with hidden targets', () => { + const originalReplace = ctx.templateSrv.replace; + ctx.templateSrv.replace = jest + .fn() + .mockImplementation( + ( + target?: string | undefined, + scopedVars?: ScopedVars | undefined, + format?: string | Function, + interpolations?: VariableInterpolation[] + ): string => { + if (target?.includes('$server')) { + return target.replaceAll('$server', scopedVars?.server?.value); + } + return originalReplace(target, scopedVars, format, interpolations); + } + ); + + const originalTargetMap = { + A: '$server.cpu', + B: 'avg(#A)', + }; + + const scopedVars = { + server: { text: 'web01', value: 'web01' }, + }; + + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: '$server.cpu', hide: true, refId: 'A' }, + { target: 'avg(#A)', refId: 'B' }, + ], + scopedVars, + }, + originalTargetMap + ); + + expect(results[0].target).toBe('avg(web01.cpu)'); + }); + + it('should not recursively replace queries that reference themselves', () => { + const originalTargetMap = { + A: 'sumSeries(carbon.test.test-host.cpuUsage, #A)', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [{ target: 'sumSeries(carbon.test.test-host.cpuUsage, #A)', refId: 'A' }], + }, + originalTargetMap + ); + expect(results[0].target).toBe( + 'sumSeries(carbon.test.test-host.cpuUsage, sumSeries(carbon.test.test-host.cpuUsage, #A))' + ); + }); + + it('should not recursively replace queries that reference themselves, but will replace nested references', () => { + const originalTargetMap = { + A: 'sumSeries(carbon.test.test-host.cpuUsage, #A, #B)', + B: 'add(carbon.test.test-host.cpuUsage, 1.5)', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { + target: 'sumSeries(carbon.test.test-host.cpuUsage, #A, #B)', + refId: 'A', + }, + { + target: 'add(carbon.test.test-host.cpuUsage, 1.5)', + refId: 'B', + }, + ], + }, + originalTargetMap + ); + expect(results[0].target).toBe( + 'sumSeries(carbon.test.test-host.cpuUsage, sumSeries(carbon.test.test-host.cpuUsage, #A, #B), add(carbon.test.test-host.cpuUsage, 1.5))' + ); + }); + + it('should ignore empty targets', () => { + const originalTargetMap = { + A: 'series1', + B: '', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [ + { target: 'series1', refId: 'A' }, + { target: '', refId: 'B' }, + ], + }, + originalTargetMap + ); + expect(results.length).toBe(1); + }); + + describe('when formatting targets', () => { + it('does not attempt to glob for one variable', () => { + const originalReplace = ctx.templateSrv.replace; + ctx.templateSrv.replace = jest + .fn() + .mockImplementation( + ( + target?: string | undefined, + scopedVars?: ScopedVars | undefined, + format?: string | Function, + interpolations?: VariableInterpolation[] + ): string => { + if (target?.includes('$metric')) { + return target.replaceAll('$metric', 'b'); + } + return originalReplace(target, scopedVars, format, interpolations); + } + ); + + const originalTargetMap = { + A: 'my.$metric.*', + }; + const results = ctx.ds.backendBuildGraphiteQueries( + { ...defaultQueryProperties, targets: [{ target: 'my.$metric.*', refId: 'A' }] }, + originalTargetMap + ); + expect(results[0].target).toEqual('my.b.*'); + }); + + it('globs for more than one variable', () => { + const originalReplace = ctx.templateSrv.replace; + ctx.templateSrv.replace = jest + .fn() + .mockImplementation( + ( + target?: string | undefined, + scopedVars?: ScopedVars | undefined, + format?: string | Function, + interpolations?: VariableInterpolation[] + ): string => { + if (target?.includes('[[metric]]')) { + return target.replaceAll('[[metric]]', '{a,b}'); + } + return originalReplace(target, scopedVars, format, interpolations); + } + ); + + const originalTargetMap = { A: 'my.[[metric]].*' }; + const results = ctx.ds.backendBuildGraphiteQueries( + { + ...defaultQueryProperties, + targets: [{ target: 'my.[[metric]].*', refId: 'A' }], + }, + originalTargetMap + ); + + expect(results[0].target).toEqual('my.{a,b}.*'); }); }); }); @@ -1108,37 +1670,58 @@ describe('graphiteDatasource', () => { }); }); -function accessScenario(name: string, url: string, fn: ({ headers }: { headers: Record }) => void) { - describe('access scenario ' + name, () => { +describe('access scenarios', () => { + describe('access scenario with proxy access', () => { const ctx = { templateSrv: getTemplateSrv(), - instanceSettings: { url: 'url', name: 'graphiteProd', jsonData: {} }, + instanceSettings: { ...instanceSettings, url: 'url', name: 'graphiteProd' }, }; - const httpOptions = { + const httpOptions: BackendSrvRequest = { headers: {}, - url, + url: '/api/datasources/proxy/1', }; describe('when using proxy mode', () => { - const options = { dashboardId: 1, panelId: 2 }; + const options = { dashboardUID: '1', panelId: 2 }; it('tracing headers should be added', () => { - ctx.instanceSettings.url = url; + ctx.instanceSettings.url = '/api/datasources/proxy/1'; const ds = new GraphiteDatasource(ctx.instanceSettings, ctx.templateSrv); ds.addTracingHeaders(httpOptions, options); - fn(httpOptions); + if (!httpOptions.headers) { + throw new Error('headers not set'); + } + expect(httpOptions.headers['X-Dashboard-Id']).toBe('1'); + expect(httpOptions.headers['X-Panel-Id']).toBe(2); }); }); }); -} -accessScenario('with proxy access', '/api/datasources/proxy/1', (httpOptions) => { - expect(httpOptions.headers['X-Dashboard-Id']).toBe(1); - expect(httpOptions.headers['X-Panel-Id']).toBe(2); -}); + describe('access scenario with direct access', () => { + const ctx = { + templateSrv: getTemplateSrv(), + instanceSettings: { ...instanceSettings, url: 'url', name: 'graphiteProd' }, + }; -accessScenario('with direct access', 'http://localhost:8080', (httpOptions) => { - expect(httpOptions.headers['X-Dashboard-Id']).toBe(undefined); - expect(httpOptions.headers['X-Panel-Id']).toBe(undefined); + const httpOptions: BackendSrvRequest = { + headers: {}, + url: 'http://localhost:8080', + }; + + describe('when using proxy mode', () => { + const options = { dashboardUid: '1', panelId: 2 }; + + it('tracing headers should be added', () => { + ctx.instanceSettings.url = 'http://localhost:8080'; + const ds = new GraphiteDatasource(ctx.instanceSettings, ctx.templateSrv); + ds.addTracingHeaders(httpOptions, options); + if (!httpOptions.headers) { + throw new Error('headers not set'); + } + expect(httpOptions.headers['X-Dashboard-Id']).toBe(undefined); + expect(httpOptions.headers['X-Panel-Id']).toBe(undefined); + }); + }); + }); }); diff --git a/public/app/plugins/datasource/graphite/datasource.ts b/public/app/plugins/datasource/graphite/datasource.ts index 93fa556c428..0a90bc776b7 100644 --- a/public/app/plugins/datasource/graphite/datasource.ts +++ b/public/app/plugins/datasource/graphite/datasource.ts @@ -47,7 +47,6 @@ import { GraphiteOptions, GraphiteQuery, GraphiteQueryImportConfiguration, - GraphiteQueryRequest, GraphiteQueryType, GraphiteType, MetricTankRequestMeta, @@ -201,48 +200,35 @@ export class GraphiteDatasource }; } - query(options: DataQueryRequest): Observable { - if (options.targets.some((target: GraphiteQuery) => target.fromAnnotations)) { - const streams: Array> = []; - - for (const target of options.targets) { - streams.push( - new Observable((subscriber) => { - this.annotationEvents(options.range, target) - .then((events) => subscriber.next({ data: [toDataFrame(events)] })) - .catch((ex) => subscriber.error(new Error(ex))) - .finally(() => subscriber.complete()); - }) - ); - } - - return merge(...streams); + filterQuery(item: GraphiteQuery): boolean { + if (!item.target && !item.fromAnnotations) { + return false; } - // Use this object to map the sanitised refID to the original - const formattedRefIdsMap: { [key: string]: string } = {}; - // Use this object to map the original refID to the original target - const originalTargetMap: { [key: string]: string } = {}; - for (const target of options.targets) { - // Sanitise the refID otherwise the Graphite query will fail - const formattedRefId = target.refId.replaceAll(' ', '_'); - formattedRefIdsMap[formattedRefId] = target.refId; - // Track the original target to ensure if we need to interpolate a series, we interpolate using the original target - // rather than the target wrapped in aliasSub e.g.: - // Suppose a query has three targets: A: metric1 B: sumSeries(#A) and C: asPercent(#A, #B) - // We want the targets to be interpolated to: A: aliasSub(metric1, "(^.*$)", "\\1 A"), B: aliasSub(sumSeries(metric1), "(^.*$)", "\\1 B") and C: asPercent(metric1, sumSeries(metric1)) - originalTargetMap[target.refId] = target.target || ''; - // Use aliasSub to include the refID in the response series name. This allows us to set the refID on the frame. - const updatedTarget = `aliasSub(${target.target}, "(^.*$)", "\\1 ${formattedRefId}")`; - target.target = updatedTarget; - } + return true; + } + // Note that we do not omit queries with the hide flag set to true to avoid breaking nested series replacement + // e.g. sumSeries(#A) where A is hidden + applyTemplateVariables(target: GraphiteQuery, scopedVars: ScopedVars) { + return { + ...target, + target: this.templateSrv.replace(target.target ?? '', scopedVars), + targetFull: this.templateSrv.replace(target.targetFull ?? '', scopedVars), + }; + } + + frontendQuery( + options: DataQueryRequest, + originalTargetMap: { [key: string]: string }, + formattedRefIdsMap: { [key: string]: string } + ): Observable { // handle the queries here const graphOptions = { from: this.translateTime(options.range.from, false, options.timezone), until: this.translateTime(options.range.to, true, options.timezone), targets: options.targets, - format: (options as GraphiteQueryRequest).format, + format: 'json', cacheTimeout: options.cacheTimeout || this.cacheTimeout, maxDataPoints: options.maxDataPoints, }; @@ -276,17 +262,123 @@ export class GraphiteDatasource ); } - addTracingHeaders( - httpOptions: BackendSrvRequest, - options: { dashboardId?: number; panelId?: number; panelPluginId?: string } - ) { - const proxyMode = !this.url.match(/^http/); + backendBuildGraphiteQueries( + options: DataQueryRequest, + originalTargetMap: { [key: string]: string } + ): GraphiteQuery[] { + const referenceTargets: Record = {}; + const finalTargets: GraphiteQuery[] = []; + let target: GraphiteQuery, targetValue, i; + + for (i = 0; i < options.targets.length; i++) { + target = options.targets[i]; + if (!target.target) { + continue; + } + + if (!target.refId) { + target.refId = this._seriesRefLetters[i]; + } + + referenceTargets[target.refId] = target.target; + } + + const seriesReferenceRegex = /\#([A-Z])/g; + + function nestedSeriesRegexReplacer(match: string, g1: string) { + // Handle the case where a query references itself to prevent infinite recursion + if (target.refId === g1) { + return referenceTargets[g1] || match; + } + + // Recursively replace all nested series references + return originalTargetMap[g1].replace(seriesReferenceRegex, nestedSeriesRegexReplacer) || match; + } + + for (i = 0; i < options.targets.length; i++) { + const targetClone = { ...options.targets[i] }; + target = options.targets[i]; + if (!targetClone.target) { + continue; + } + + targetValue = this.templateSrv.replace( + referenceTargets[target.refId].replace(seriesReferenceRegex, nestedSeriesRegexReplacer), + options.scopedVars + ); + + targetClone.target = targetValue; + if (this.isMetricTank) { + targetClone.isMetricTank = true; + } + + if (!targetClone.hide) { + finalTargets.push(targetClone); + } + } + + return finalTargets; + } + + query(options: DataQueryRequest): Observable { + if (options.targets.some((target: GraphiteQuery) => target.fromAnnotations)) { + const streams: Array> = []; + + for (const target of options.targets) { + streams.push( + new Observable((subscriber) => { + this.annotationEvents(options.range, target) + .then((events) => subscriber.next({ data: [toDataFrame(events)] })) + .catch((ex) => subscriber.error(new Error(ex))) + .finally(() => subscriber.complete()); + }) + ); + } + + return merge(...streams); + } + + // Use this object to map the sanitised refID to the original + const formattedRefIdsMap: { [key: string]: string } = {}; + // Use this object to map the original refID to the original target + const originalTargetMap: { [key: string]: string } = {}; + for (const target of options.targets) { + // Sanitise the refID otherwise the Graphite query will fail + const formattedRefId = target.refId.replaceAll(' ', '_'); + formattedRefIdsMap[formattedRefId] = target.refId; + // Track the original target to ensure if we need to interpolate a series, we interpolate using the original target + // rather than the target wrapped in aliasSub e.g.: + // Suppose a query has three targets: A: metric1 B: sumSeries(#A) and C: asPercent(#A, #B) + // We want the targets to be interpolated to: A: aliasSub(metric1, "(^.*$)", "\\1 A"), B: aliasSub(sumSeries(metric1), "(^.*$)", "\\1 B") and C: asPercent(metric1, sumSeries(metric1)) + originalTargetMap[target.refId] = target.target || ''; + + // We only need to alias queries in frontend mode + if (!config.featureToggles.graphiteBackendMode) { + // Use aliasSub to include the refID in the response series name. This allows us to set the refID on the frame. + const updatedTarget = `aliasSub(${target.target}, "(^.*$)", "\\1 ${formattedRefId}")`; + target.target = updatedTarget; + } + } + + if (config.featureToggles.graphiteBackendMode) { + const graphiteQueries = this.backendBuildGraphiteQueries(options, originalTargetMap); + + options.targets = graphiteQueries; + + return super.query(options); + } + + return this.frontendQuery(options, originalTargetMap, formattedRefIdsMap); + } + + addTracingHeaders(httpOptions: BackendSrvRequest, options: Partial>) { + const proxyMode = !this.url?.match(/^http/); if (!httpOptions.headers) { httpOptions.headers = {}; } if (proxyMode) { - if (options.dashboardId) { - httpOptions.headers['X-Dashboard-Id'] = options.dashboardId; + if (options.dashboardUID) { + httpOptions.headers['X-Dashboard-Id'] = options.dashboardUID; } if (options.panelId) { httpOptions.headers['X-Panel-Id'] = options.panelId; @@ -660,6 +752,9 @@ export class GraphiteDatasource let result: MetricFindValue[]; if (queryType === GraphiteQueryType.Value) { + if (!data.data || data.data.length === 0) { + return Promise.resolve([]); + } result = data.data[0].fields[1].values .filter((f?: number) => !!f) .map((v: number) => ({ @@ -1014,6 +1109,7 @@ export class GraphiteDatasource ); } + // Can be removed when the frontend query path is removed buildGraphiteParams(options: any, originalTargetMap: { [key: string]: string }, scopedVars?: ScopedVars): string[] { const graphiteOptions = ['from', 'until', 'rawData', 'format', 'maxDataPoints', 'cacheTimeout']; const cleanOptions = [], @@ -1045,7 +1141,7 @@ export class GraphiteDatasource const regex = /\#([A-Z])/g; - function nestedSeriesRegexReplacer(match: string, g1: string | number) { + function nestedSeriesRegexReplacer(match: string, g1: string) { // Handle the case where a query references itself to prevent infinite recursion if (target.refId === g1) { return targets[g1] || match; diff --git a/public/app/plugins/datasource/graphite/types.ts b/public/app/plugins/datasource/graphite/types.ts index 338f49f5922..40e3e698bf9 100644 --- a/public/app/plugins/datasource/graphite/types.ts +++ b/public/app/plugins/datasource/graphite/types.ts @@ -17,6 +17,7 @@ export interface GraphiteQuery extends DataQuery { targetFull?: string; tags?: string[]; fromAnnotations?: boolean; + isMetricTank?: boolean; } export interface GraphiteOptions extends DataSourceJsonData {