From 962d4daabd4befeb78c0a6688a54cac03a4cf96c Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Mon, 8 Jul 2019 14:30:02 +0200 Subject: [PATCH 1/5] Api: Fix auth tokens returning wrong seenAt value (#17980) Changes so a non-set seenAt property gets the createdAt value. Fixes #17953 --- pkg/api/user_token.go | 11 +++++++++-- pkg/api/user_token_test.go | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/api/user_token.go b/pkg/api/user_token.go index 9c29e6ec89c..4cf010e8ace 100644 --- a/pkg/api/user_token.go +++ b/pkg/api/user_token.go @@ -84,6 +84,13 @@ func (server *HTTPServer) getUserAuthTokensInternal(c *models.ReqContext, userID } } + createdAt := time.Unix(token.CreatedAt, 0) + seenAt := time.Unix(token.SeenAt, 0) + + if token.SeenAt == 0 { + seenAt = createdAt + } + result = append(result, &dtos.UserToken{ Id: token.Id, IsActive: isActive, @@ -93,8 +100,8 @@ func (server *HTTPServer) getUserAuthTokensInternal(c *models.ReqContext, userID OperatingSystemVersion: osVersion, Browser: client.UserAgent.Family, BrowserVersion: browserVersion, - CreatedAt: time.Unix(token.CreatedAt, 0), - SeenAt: time.Unix(token.SeenAt, 0), + CreatedAt: createdAt, + SeenAt: seenAt, }) } diff --git a/pkg/api/user_token_test.go b/pkg/api/user_token_test.go index ad5384ecdaf..6b8ebcf64e2 100644 --- a/pkg/api/user_token_test.go +++ b/pkg/api/user_token_test.go @@ -124,7 +124,7 @@ func TestUserTokenApiEndpoint(t *testing.T) { ClientIp: "127.0.0.2", UserAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1", CreatedAt: time.Now().Unix(), - SeenAt: time.Now().Unix(), + SeenAt: 0, }, } sc.userAuthTokenService.GetUserTokensProvider = func(ctx context.Context, userId int64) ([]*m.UserToken, error) { @@ -154,7 +154,7 @@ func TestUserTokenApiEndpoint(t *testing.T) { So(resultTwo.Get("isActive").MustBool(), ShouldBeFalse) So(resultTwo.Get("clientIp").MustString(), ShouldEqual, "127.0.0.2") So(resultTwo.Get("createdAt").MustString(), ShouldEqual, time.Unix(tokens[1].CreatedAt, 0).Format(time.RFC3339)) - So(resultTwo.Get("seenAt").MustString(), ShouldEqual, time.Unix(tokens[1].SeenAt, 0).Format(time.RFC3339)) + So(resultTwo.Get("seenAt").MustString(), ShouldEqual, time.Unix(tokens[1].CreatedAt, 0).Format(time.RFC3339)) So(resultTwo.Get("device").MustString(), ShouldEqual, "iPhone") So(resultTwo.Get("browser").MustString(), ShouldEqual, "Mobile Safari") From 8f5df8016115242cf94589bb33a10470cf238d99 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 8 Jul 2019 07:20:29 -0700 Subject: [PATCH 2/5] Chore: Removes custom debounce utility in favor of lodash/debounce (#17977) --- public/app/features/explore/utils/debounce.ts | 14 -------------- .../editor/KustoQueryField.tsx | 4 +--- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 public/app/features/explore/utils/debounce.ts diff --git a/public/app/features/explore/utils/debounce.ts b/public/app/features/explore/utils/debounce.ts deleted file mode 100644 index a7c9450a6c1..00000000000 --- a/public/app/features/explore/utils/debounce.ts +++ /dev/null @@ -1,14 +0,0 @@ -// Based on underscore.js debounce() -export default function debounce(func, wait) { - let timeout; - return function(this: any) { - const context = this; - const args = arguments; - const later = () => { - timeout = null; - func.apply(context, args); - }; - clearTimeout(timeout); - timeout = setTimeout(later, wait); - }; -} diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/editor/KustoQueryField.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/editor/KustoQueryField.tsx index 6cd813cac41..fc4d1a44822 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/editor/KustoQueryField.tsx +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/editor/KustoQueryField.tsx @@ -3,9 +3,7 @@ import _ from 'lodash'; import Plain from 'slate-plain-serializer'; import QueryField from './query_field'; -// import debounce from './utils/debounce'; -// import {getNextCharacter} from './utils/dom'; -import debounce from 'app/features/explore/utils/debounce'; +import debounce from 'lodash/debounce'; import { getNextCharacter } from 'app/features/explore/utils/dom'; import { KEYWORDS, functionTokens, operatorTokens, grafanaMacros } from './kusto/kusto'; From 246358344cf04d0f49012aa291968375ef58f2aa Mon Sep 17 00:00:00 2001 From: kay delaney <45561153+kaydelaney@users.noreply.github.com> Date: Mon, 8 Jul 2019 16:14:48 +0100 Subject: [PATCH 3/5] Explore: Adds support for new loki 'start' and 'end' params for labels endpoint (#17512) * Explore: Adds support for new loki 'start' and 'end' params for labels endpoint Also initializes absoluteRange when explore is initialized Closes #16788 * Explore: Dispatches updateTimeRangeAction instead of passing absoluteRange when initializing Also removes dependency on sinon from loki language provider test * Loki: Refactors transformation of absolute time range to URL params into small utility function * Makes use of rangeToParams() util function in loki language provider test Also updates LanguageProvider.request() interface so that url should be type string, and adds optional params argument --- public/app/features/explore/QueryRow.tsx | 7 +- public/app/features/explore/state/actions.ts | 1 + .../loki/components/LokiQueryField.tsx | 3 +- .../loki/components/LokiQueryFieldForm.tsx | 6 +- .../loki/components/useLokiLabels.test.ts | 25 ++++- .../loki/components/useLokiLabels.ts | 6 +- .../loki/components/useLokiSyntax.test.ts | 18 +++- .../loki/components/useLokiSyntax.ts | 10 +- .../app/plugins/datasource/loki/datasource.ts | 5 +- .../datasource/loki/language_provider.test.ts | 100 ++++++++++++++---- .../datasource/loki/language_provider.ts | 35 +++--- public/app/types/explore.ts | 2 +- 12 files changed, 163 insertions(+), 55 deletions(-) diff --git a/public/app/features/explore/QueryRow.tsx b/public/app/features/explore/QueryRow.tsx index 5b5361fcff7..7dc1b3bcd42 100644 --- a/public/app/features/explore/QueryRow.tsx +++ b/public/app/features/explore/QueryRow.tsx @@ -13,7 +13,7 @@ import { changeQuery, modifyQueries, runQueries, addQueryRow } from './state/act // Types import { StoreState } from 'app/types'; -import { TimeRange } from '@grafana/data'; +import { TimeRange, AbsoluteTimeRange } from '@grafana/data'; import { DataQuery, DataSourceApi, QueryFixAction, DataSourceStatus, PanelData, DataQueryError } from '@grafana/ui'; import { HistoryItem, ExploreItemState, ExploreId, ExploreMode } from 'app/types/explore'; import { Emitter } from 'app/core/utils/emitter'; @@ -38,6 +38,7 @@ interface QueryRowProps extends PropsFromParent { query: DataQuery; modifyQueries: typeof modifyQueries; range: TimeRange; + absoluteRange: AbsoluteTimeRange; removeQueryRowAction: typeof removeQueryRowAction; runQueries: typeof runQueries; queryResponse: PanelData; @@ -116,6 +117,7 @@ export class QueryRow extends PureComponent { query, exploreEvents, range, + absoluteRange, datasourceStatus, queryResponse, latency, @@ -148,6 +150,7 @@ export class QueryRow extends PureComponent { onChange={this.onChange} panelData={null} queryResponse={queryResponse} + absoluteRange={absoluteRange} /> ) : ( = ({ }) => { const { isSyntaxReady, setActiveOption, refreshLabels, ...syntaxProps } = useLokiSyntax( datasource.languageProvider, - datasourceStatus + datasourceStatus, + otherProps.absoluteRange ); return ( diff --git a/public/app/plugins/datasource/loki/components/LokiQueryFieldForm.tsx b/public/app/plugins/datasource/loki/components/LokiQueryFieldForm.tsx index a23cd86ad58..8646a7c6849 100644 --- a/public/app/plugins/datasource/loki/components/LokiQueryFieldForm.tsx +++ b/public/app/plugins/datasource/loki/components/LokiQueryFieldForm.tsx @@ -17,6 +17,7 @@ import BracesPlugin from 'app/features/explore/slate-plugins/braces'; import { LokiQuery } from '../types'; import { TypeaheadOutput, HistoryItem } from 'app/types/explore'; import { DataSourceApi, ExploreQueryFieldProps, DataSourceStatus } from '@grafana/ui'; +import { AbsoluteTimeRange } from '@grafana/data'; function getChooserText(hasSyntax: boolean, hasLogLabels: boolean, datasourceStatus: DataSourceStatus) { if (datasourceStatus === DataSourceStatus.Disconnected) { @@ -70,6 +71,7 @@ export interface LokiQueryFieldFormProps extends ExploreQueryFieldProps void; onLabelsRefresh?: () => void; } @@ -123,7 +125,7 @@ export class LokiQueryFieldForm extends React.PureComponent { it('should refresh labels', async () => { @@ -10,6 +11,10 @@ describe('useLokiLabels hook', () => { }; const languageProvider = new LanguageProvider(datasource); const logLabelOptionsMock = ['Holy mock!']; + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560153109000, + }; languageProvider.refreshLogLabels = () => { languageProvider.logLabelOptions = logLabelOptionsMock; @@ -17,7 +22,7 @@ describe('useLokiLabels hook', () => { }; const { result, waitForNextUpdate } = renderHook(() => - useLokiLabels(languageProvider, true, [], DataSourceStatus.Connected, DataSourceStatus.Connected) + useLokiLabels(languageProvider, true, [], rangeMock, DataSourceStatus.Connected, DataSourceStatus.Connected) ); act(() => result.current.refreshLabels()); expect(result.current.logLabelOptions).toEqual([]); @@ -29,26 +34,38 @@ describe('useLokiLabels hook', () => { const datasource = { metadataRequest: () => ({ data: { data: [] as any[] } }), }; + + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560153109000, + }; + const languageProvider = new LanguageProvider(datasource); languageProvider.refreshLogLabels = jest.fn(); renderHook(() => - useLokiLabels(languageProvider, true, [], DataSourceStatus.Connected, DataSourceStatus.Disconnected) + useLokiLabels(languageProvider, true, [], rangeMock, DataSourceStatus.Connected, DataSourceStatus.Disconnected) ); expect(languageProvider.refreshLogLabels).toBeCalledTimes(1); - expect(languageProvider.refreshLogLabels).toBeCalledWith(true); + expect(languageProvider.refreshLogLabels).toBeCalledWith(rangeMock, true); }); it('should not force refresh labels after a connect', () => { const datasource = { metadataRequest: () => ({ data: { data: [] as any[] } }), }; + + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560153109000, + }; + const languageProvider = new LanguageProvider(datasource); languageProvider.refreshLogLabels = jest.fn(); renderHook(() => - useLokiLabels(languageProvider, true, [], DataSourceStatus.Disconnected, DataSourceStatus.Connected) + useLokiLabels(languageProvider, true, [], rangeMock, DataSourceStatus.Disconnected, DataSourceStatus.Connected) ); expect(languageProvider.refreshLogLabels).not.toBeCalled(); diff --git a/public/app/plugins/datasource/loki/components/useLokiLabels.ts b/public/app/plugins/datasource/loki/components/useLokiLabels.ts index b73f4a3d916..487814501d1 100644 --- a/public/app/plugins/datasource/loki/components/useLokiLabels.ts +++ b/public/app/plugins/datasource/loki/components/useLokiLabels.ts @@ -1,5 +1,6 @@ import { useState, useEffect } from 'react'; import { DataSourceStatus } from '@grafana/ui/src/types/datasource'; +import { AbsoluteTimeRange } from '@grafana/data'; import LokiLanguageProvider from 'app/plugins/datasource/loki/language_provider'; import { CascaderOption } from 'app/plugins/datasource/loki/components/LokiQueryFieldForm'; @@ -17,6 +18,7 @@ export const useLokiLabels = ( languageProvider: LokiLanguageProvider, languageProviderInitialised: boolean, activeOption: CascaderOption[], + absoluteRange: AbsoluteTimeRange, datasourceStatus: DataSourceStatus, initialDatasourceStatus?: DataSourceStatus // used for test purposes ) => { @@ -32,14 +34,14 @@ export const useLokiLabels = ( // Async const fetchOptionValues = async (option: string) => { - await languageProvider.fetchLabelValues(option); + await languageProvider.fetchLabelValues(option, absoluteRange); if (mounted.current) { setLogLabelOptions(languageProvider.logLabelOptions); } }; const tryLabelsRefresh = async () => { - await languageProvider.refreshLogLabels(shouldForceRefreshLabels); + await languageProvider.refreshLogLabels(absoluteRange, shouldForceRefreshLabels); if (mounted.current) { setRefreshLabels(false); diff --git a/public/app/plugins/datasource/loki/components/useLokiSyntax.test.ts b/public/app/plugins/datasource/loki/components/useLokiSyntax.test.ts index bee092ecd87..99e7904c347 100644 --- a/public/app/plugins/datasource/loki/components/useLokiSyntax.test.ts +++ b/public/app/plugins/datasource/loki/components/useLokiSyntax.test.ts @@ -1,5 +1,6 @@ import { renderHook, act } from 'react-hooks-testing-library'; import { DataSourceStatus } from '@grafana/ui/src/types/datasource'; +import { AbsoluteTimeRange } from '@grafana/data'; import LanguageProvider from 'app/plugins/datasource/loki/language_provider'; import { useLokiSyntax } from './useLokiSyntax'; @@ -14,6 +15,11 @@ describe('useLokiSyntax hook', () => { const logLabelOptionsMock2 = ['Mock the hell?!']; const logLabelOptionsMock3 = ['Oh my mock!']; + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560163909000, + }; + languageProvider.refreshLogLabels = () => { languageProvider.logLabelOptions = logLabelOptionsMock; return Promise.resolve(); @@ -30,7 +36,9 @@ describe('useLokiSyntax hook', () => { }; it('should provide Loki syntax when used', async () => { - const { result, waitForNextUpdate } = renderHook(() => useLokiSyntax(languageProvider, DataSourceStatus.Connected)); + const { result, waitForNextUpdate } = renderHook(() => + useLokiSyntax(languageProvider, DataSourceStatus.Connected, rangeMock) + ); expect(result.current.syntax).toEqual(null); await waitForNextUpdate(); @@ -39,7 +47,9 @@ describe('useLokiSyntax hook', () => { }); it('should fetch labels on first call', async () => { - const { result, waitForNextUpdate } = renderHook(() => useLokiSyntax(languageProvider, DataSourceStatus.Connected)); + const { result, waitForNextUpdate } = renderHook(() => + useLokiSyntax(languageProvider, DataSourceStatus.Connected, rangeMock) + ); expect(result.current.isSyntaxReady).toBeFalsy(); expect(result.current.logLabelOptions).toEqual([]); @@ -50,7 +60,9 @@ describe('useLokiSyntax hook', () => { }); it('should try to fetch missing options when active option changes', async () => { - const { result, waitForNextUpdate } = renderHook(() => useLokiSyntax(languageProvider, DataSourceStatus.Connected)); + const { result, waitForNextUpdate } = renderHook(() => + useLokiSyntax(languageProvider, DataSourceStatus.Connected, rangeMock) + ); await waitForNextUpdate(); expect(result.current.logLabelOptions).toEqual(logLabelOptionsMock2); diff --git a/public/app/plugins/datasource/loki/components/useLokiSyntax.ts b/public/app/plugins/datasource/loki/components/useLokiSyntax.ts index a48afe19ca1..7faa5a6fb24 100644 --- a/public/app/plugins/datasource/loki/components/useLokiSyntax.ts +++ b/public/app/plugins/datasource/loki/components/useLokiSyntax.ts @@ -2,7 +2,7 @@ import { useState, useEffect } from 'react'; // @ts-ignore import Prism from 'prismjs'; import { DataSourceStatus } from '@grafana/ui/src/types/datasource'; - +import { AbsoluteTimeRange } from '@grafana/data'; import LokiLanguageProvider from 'app/plugins/datasource/loki/language_provider'; import { useLokiLabels } from 'app/plugins/datasource/loki/components/useLokiLabels'; import { CascaderOption } from 'app/plugins/datasource/loki/components/LokiQueryFieldForm'; @@ -15,7 +15,11 @@ const PRISM_SYNTAX = 'promql'; * @param languageProvider * @description Initializes given language provider, exposes Loki syntax and enables loading label option values */ -export const useLokiSyntax = (languageProvider: LokiLanguageProvider, datasourceStatus: DataSourceStatus) => { +export const useLokiSyntax = ( + languageProvider: LokiLanguageProvider, + datasourceStatus: DataSourceStatus, + absoluteRange: AbsoluteTimeRange +) => { const mounted = useRefMounted(); // State const [languageProviderInitialized, setLanguageProviderInitilized] = useState(false); @@ -32,11 +36,13 @@ export const useLokiSyntax = (languageProvider: LokiLanguageProvider, datasource languageProvider, languageProviderInitialized, activeOption, + absoluteRange, datasourceStatus ); // Async const initializeLanguageProvider = async () => { + languageProvider.initialRange = absoluteRange; await languageProvider.start(); Prism.languages[PRISM_SYNTAX] = languageProvider.getSyntax(); if (mounted.current) { diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 28a0857e377..bd4080b061b 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -78,6 +78,7 @@ export class LokiDatasource extends DataSourceApi { ...options, url, }; + return this.backendSrv.datasourceRequest(req); } @@ -254,10 +255,10 @@ export class LokiDatasource extends DataSourceApi { return this.languageProvider.importQueries(queries, originMeta.id); } - metadataRequest(url: string) { + metadataRequest(url: string, params?: any) { // HACK to get label values for {job=|}, will be replaced when implementing LokiQueryField const apiUrl = url.replace('v1', 'prom'); - return this._request(apiUrl, { silent: true }).then((res: DataQueryResponse) => { + return this._request(apiUrl, params, { silent: true }).then((res: DataQueryResponse) => { const data: any = { data: { data: res.data.values || [] } }; return data; }); diff --git a/public/app/plugins/datasource/loki/language_provider.test.ts b/public/app/plugins/datasource/loki/language_provider.test.ts index 301fbb30fcd..986f5335821 100644 --- a/public/app/plugins/datasource/loki/language_provider.test.ts +++ b/public/app/plugins/datasource/loki/language_provider.test.ts @@ -1,7 +1,8 @@ // @ts-ignore import Plain from 'slate-plain-serializer'; -import LanguageProvider, { LABEL_REFRESH_INTERVAL } from './language_provider'; +import LanguageProvider, { LABEL_REFRESH_INTERVAL, rangeToParams } from './language_provider'; +import { AbsoluteTimeRange } from '@grafana/data'; import { advanceTo, clear, advanceBy } from 'jest-date-mock'; import { beforeEach } from 'test/lib/common'; import { DataQueryResponseData } from '@grafana/ui'; @@ -11,8 +12,13 @@ describe('Language completion provider', () => { metadataRequest: () => ({ data: { data: [] as DataQueryResponseData[] } }), }; + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560163909000, + }; + describe('empty query suggestions', () => { - it('returns no suggestions on emtpty context', () => { + it('returns no suggestions on empty context', () => { const instance = new LanguageProvider(datasource); const value = Plain.deserialize(''); const result = instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] }); @@ -21,7 +27,7 @@ describe('Language completion provider', () => { expect(result.suggestions.length).toEqual(0); }); - it('returns default suggestions with history on emtpty context when history was provided', () => { + it('returns default suggestions with history on empty context when history was provided', () => { const instance = new LanguageProvider(datasource); const value = Plain.deserialize(''); const history = [ @@ -29,7 +35,10 @@ describe('Language completion provider', () => { query: { refId: '1', expr: '{app="foo"}' }, }, ]; - const result = instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] }, { history }); + const result = instance.provideCompletionItems( + { text: '', prefix: '', value, wrapperClasses: [] }, + { history, absoluteRange: rangeMock } + ); expect(result.context).toBeUndefined(); expect(result.refresher).toBeUndefined(); expect(result.suggestions).toMatchObject([ @@ -79,64 +88,102 @@ describe('Language completion provider', () => { anchorOffset: 1, }); const valueWithSelection = value.change().select(range).value; - const result = instance.provideCompletionItems({ - text: '', - prefix: '', - wrapperClasses: ['context-labels'], - value: valueWithSelection, - }); + const result = instance.provideCompletionItems( + { + text: '', + prefix: '', + wrapperClasses: ['context-labels'], + value: valueWithSelection, + }, + { absoluteRange: rangeMock } + ); expect(result.context).toBe('context-labels'); expect(result.suggestions).toEqual([{ items: [{ label: 'job' }, { label: 'namespace' }], label: 'Labels' }]); }); }); }); +describe('Request URL', () => { + it('should contain range params', async () => { + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560163909000, + }; + + const datasourceWithLabels = { + metadataRequest: url => { + if (url.slice(0, 15) === '/api/prom/label') { + return { data: { data: ['other'] } }; + } else { + return { data: { data: [] } }; + } + }, + }; + + const datasourceSpy = jest.spyOn(datasourceWithLabels, 'metadataRequest'); + + const instance = new LanguageProvider(datasourceWithLabels, { initialRange: rangeMock }); + await instance.refreshLogLabels(rangeMock, true); + const expectedUrl = '/api/prom/label'; + expect(datasourceSpy).toHaveBeenCalledWith(expectedUrl, rangeToParams(rangeMock)); + }); +}); + describe('Query imports', () => { const datasource = { metadataRequest: () => ({ data: { data: [] as DataQueryResponseData[] } }), }; + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560163909000, + }; + it('returns empty queries for unknown origin datasource', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(datasource, { initialRange: rangeMock }); const result = await instance.importQueries([{ refId: 'bar', expr: 'foo' }], 'unknown'); expect(result).toEqual([{ refId: 'bar', expr: '' }]); }); describe('prometheus query imports', () => { it('returns empty query from metric-only query', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(datasource, { initialRange: rangeMock }); const result = await instance.importPrometheusQuery('foo'); expect(result).toEqual(''); }); it('returns empty query from selector query if label is not available', async () => { const datasourceWithLabels = { - metadataRequest: (url: string) => - url === '/api/prom/label' ? { data: { data: ['other'] } } : { data: { data: [] as DataQueryResponseData[] } }, + metadataRequest: url => + url.slice(0, 15) === '/api/prom/label' + ? { data: { data: ['other'] } } + : { data: { data: [] as DataQueryResponseData[] } }, }; - const instance = new LanguageProvider(datasourceWithLabels); + const instance = new LanguageProvider(datasourceWithLabels, { initialRange: rangeMock }); const result = await instance.importPrometheusQuery('{foo="bar"}'); expect(result).toEqual('{}'); }); it('returns selector query from selector query with common labels', async () => { const datasourceWithLabels = { - metadataRequest: (url: string) => - url === '/api/prom/label' ? { data: { data: ['foo'] } } : { data: { data: [] as DataQueryResponseData[] } }, + metadataRequest: url => + url.slice(0, 15) === '/api/prom/label' + ? { data: { data: ['foo'] } } + : { data: { data: [] as DataQueryResponseData[] } }, }; - const instance = new LanguageProvider(datasourceWithLabels); + const instance = new LanguageProvider(datasourceWithLabels, { initialRange: rangeMock }); const result = await instance.importPrometheusQuery('metric{foo="bar",baz="42"}'); expect(result).toEqual('{foo="bar"}'); }); it('returns selector query from selector query with all labels if logging label list is empty', async () => { const datasourceWithLabels = { - metadataRequest: (url: string) => - url === '/api/prom/label' + metadataRequest: url => + url.slice(0, 15) === '/api/prom/label' ? { data: { data: [] as DataQueryResponseData[] } } : { data: { data: [] as DataQueryResponseData[] } }, }; - const instance = new LanguageProvider(datasourceWithLabels); + const instance = new LanguageProvider(datasourceWithLabels, { initialRange: rangeMock }); const result = await instance.importPrometheusQuery('metric{foo="bar",baz="42"}'); expect(result).toEqual('{baz="42",foo="bar"}'); }); @@ -149,6 +196,11 @@ describe('Labels refresh', () => { }; const instance = new LanguageProvider(datasource); + const rangeMock: AbsoluteTimeRange = { + from: 1560153109000, + to: 1560163909000, + }; + beforeEach(() => { instance.fetchLogLabels = jest.fn(); }); @@ -157,18 +209,20 @@ describe('Labels refresh', () => { jest.clearAllMocks(); clear(); }); + it("should not refresh labels if refresh interval hasn't passed", () => { advanceTo(new Date(2019, 1, 1, 0, 0, 0)); instance.logLabelFetchTs = Date.now(); advanceBy(LABEL_REFRESH_INTERVAL / 2); - instance.refreshLogLabels(); + instance.refreshLogLabels(rangeMock); expect(instance.fetchLogLabels).not.toBeCalled(); }); + it('should refresh labels if refresh interval passed', () => { advanceTo(new Date(2019, 1, 1, 0, 0, 0)); instance.logLabelFetchTs = Date.now(); advanceBy(LABEL_REFRESH_INTERVAL + 1); - instance.refreshLogLabels(); + instance.refreshLogLabels(rangeMock); expect(instance.fetchLogLabels).toBeCalled(); }); }); diff --git a/public/app/plugins/datasource/loki/language_provider.ts b/public/app/plugins/datasource/loki/language_provider.ts index 0c4877066d7..f465d9c022a 100644 --- a/public/app/plugins/datasource/loki/language_provider.ts +++ b/public/app/plugins/datasource/loki/language_provider.ts @@ -15,16 +15,18 @@ import { HistoryItem, } from 'app/types/explore'; import { LokiQuery } from './types'; -import { dateTime } from '@grafana/data'; +import { dateTime, AbsoluteTimeRange } from '@grafana/data'; import { PromQuery } from '../prometheus/types'; const DEFAULT_KEYS = ['job', 'namespace']; const EMPTY_SELECTOR = '{}'; const HISTORY_ITEM_COUNT = 10; const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h +const NS_IN_MS = 1_000_000; export const LABEL_REFRESH_INTERVAL = 1000 * 30; // 30sec const wrapLabel = (label: string) => ({ label }); +export const rangeToParams = (range: AbsoluteTimeRange) => ({ start: range.from * NS_IN_MS, end: range.to * NS_IN_MS }); type LokiHistoryItem = HistoryItem; @@ -50,6 +52,7 @@ export default class LokiLanguageProvider extends LanguageProvider { logLabelOptions: any[]; logLabelFetchTs?: number; started: boolean; + initialRange: AbsoluteTimeRange; constructor(datasource: any, initialValues?: any) { super(); @@ -67,13 +70,13 @@ export default class LokiLanguageProvider extends LanguageProvider { return syntax; } - request = (url: string) => { - return this.datasource.metadataRequest(url); + request = (url: string, params?: any) => { + return this.datasource.metadataRequest(url, params); }; start = () => { if (!this.startTask) { - this.startTask = this.fetchLogLabels(); + this.startTask = this.fetchLogLabels(this.initialRange); } return this.startTask; }; @@ -120,7 +123,10 @@ export default class LokiLanguageProvider extends LanguageProvider { return { suggestions }; } - getLabelCompletionItems({ text, wrapperClasses, labelKey, value }: TypeaheadInput): TypeaheadOutput { + getLabelCompletionItems( + { text, wrapperClasses, labelKey, value }: TypeaheadInput, + { absoluteRange }: any + ): TypeaheadOutput { let context: string; let refresher: Promise = null; const suggestions: CompletionItemGroup[] = []; @@ -146,7 +152,7 @@ export default class LokiLanguageProvider extends LanguageProvider { items: labelValues.map(wrapLabel), }); } else { - refresher = this.fetchLabelValues(labelKey); + refresher = this.fetchLabelValues(labelKey, absoluteRange); } } } else { @@ -206,7 +212,7 @@ export default class LokiLanguageProvider extends LanguageProvider { if (existingKeys && existingKeys.length > 0) { // Check for common labels for (const key in labels) { - if (existingKeys && existingKeys.indexOf(key) > -1) { + if (existingKeys && existingKeys.includes(key)) { // Should we check for label value equality here? labelsToKeep[key] = labels[key]; } @@ -227,11 +233,12 @@ export default class LokiLanguageProvider extends LanguageProvider { return ''; } - async fetchLogLabels(): Promise { + async fetchLogLabels(absoluteRange: AbsoluteTimeRange): Promise { const url = '/api/prom/label'; try { this.logLabelFetchTs = Date.now(); - const res = await this.request(url); + + const res = await this.request(url, rangeToParams(absoluteRange)); const body = await (res.data || res.json()); const labelKeys = body.data.slice().sort(); this.labelKeys = { @@ -244,7 +251,7 @@ export default class LokiLanguageProvider extends LanguageProvider { return Promise.all( labelKeys .filter((key: string) => DEFAULT_KEYS.indexOf(key) > -1) - .map((key: string) => this.fetchLabelValues(key)) + .map((key: string) => this.fetchLabelValues(key, absoluteRange)) ); } catch (e) { console.error(e); @@ -252,16 +259,16 @@ export default class LokiLanguageProvider extends LanguageProvider { return []; } - async refreshLogLabels(forceRefresh?: boolean) { + async refreshLogLabels(absoluteRange: AbsoluteTimeRange, forceRefresh?: boolean) { if ((this.labelKeys && Date.now() - this.logLabelFetchTs > LABEL_REFRESH_INTERVAL) || forceRefresh) { - await this.fetchLogLabels(); + await this.fetchLogLabels(absoluteRange); } } - async fetchLabelValues(key: string) { + async fetchLabelValues(key: string, absoluteRange: AbsoluteTimeRange) { const url = `/api/prom/label/${key}/values`; try { - const res = await this.request(url); + const res = await this.request(url, rangeToParams(absoluteRange)); const body = await (res.data || res.json()); const values = body.data.slice().sort(); diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index d816255ba84..5a2ddd1bc72 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -286,7 +286,7 @@ export interface HistoryItem { export abstract class LanguageProvider { datasource: any; - request: (url: any) => Promise; + request: (url: string, params?: any) => Promise; /** * Returns startTask that resolves with a task list when main syntax is loaded. * Task list consists of secondary promises that load more detailed language features. From c284a11a957fcbd64b6eea896202e63a1fd7dfec Mon Sep 17 00:00:00 2001 From: Martin Chodur Date: Mon, 8 Jul 2019 17:48:09 +0200 Subject: [PATCH 4/5] Prometheus: added time range filter to series labels query (#16851) * fix datasource/prometheus: added time range filter to prom series labels * fix datasource/prometheus: added missing method to test mock --- .../prometheus/language_provider.ts | 22 ++++++++++++++++--- .../specs/language_provider.test.ts | 1 + 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/prometheus/language_provider.ts b/public/app/plugins/datasource/prometheus/language_provider.ts index 294ad9b2880..86cdb57f02a 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.ts @@ -42,6 +42,7 @@ export function addHistoryMetadata(item: CompletionItem, history: any[]): Comple export default class PromQlLanguageProvider extends LanguageProvider { histogramMetrics?: string[]; + timeRange?: { start: number; end: number }; labelKeys?: { [index: string]: string[] }; // metric -> [labelKey,...] labelValues?: { [index: string]: { [index: string]: string[] } }; // metric -> labelKey -> [labelValue,...] metrics?: string[]; @@ -52,6 +53,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { this.datasource = datasource; this.histogramMetrics = []; + this.timeRange = { start: 0, end: 0 }; this.labelKeys = {}; this.labelValues = {}; this.metrics = []; @@ -205,6 +207,18 @@ export default class PromQlLanguageProvider extends LanguageProvider { }; } + roundToMinutes(seconds: number): number { + return Math.floor(seconds / 60); + } + + timeRangeChanged(): boolean { + const dsRange = this.datasource.getTimeRange(); + return ( + this.roundToMinutes(dsRange.end) !== this.roundToMinutes(this.timeRange.end) || + this.roundToMinutes(dsRange.start) !== this.roundToMinutes(this.timeRange.start) + ); + } + getAggregationCompletionItems({ value }: TypeaheadInput): TypeaheadOutput { const refresher: Promise = null; const suggestions: CompletionItemGroup[] = []; @@ -253,7 +267,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { const selector = parseSelector(selectorString, selectorString.length - 2).selector; const labelKeys = this.labelKeys[selector]; - if (labelKeys) { + if (labelKeys && !this.timeRangeChanged()) { suggestions.push({ label: 'Labels', items: labelKeys.map(wrapLabel) }); } else { result.refresher = this.fetchSeriesLabels(selector); @@ -304,7 +318,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { } // Query labels for selector - if (selector && !this.labelValues[selector]) { + if (selector && (!this.labelValues[selector] || this.timeRangeChanged())) { if (selector === EMPTY_SELECTOR) { // Query label values for default labels refresher = Promise.all(DEFAULT_KEYS.map(key => this.fetchLabelValues(key))); @@ -332,10 +346,12 @@ export default class PromQlLanguageProvider extends LanguageProvider { fetchSeriesLabels = async (name: string, withName?: boolean) => { try { - const data = await this.request(`/api/v1/series?match[]=${name}`); + const tRange = this.datasource.getTimeRange(); + const data = await this.request(`/api/v1/series?match[]=${name}&start=${tRange['start']}&end=${tRange['end']}`); const { keys, values } = processLabels(data, withName); this.labelKeys[name] = keys; this.labelValues[name] = values; + this.timeRange = tRange; } catch (e) { console.error(e); } diff --git a/public/app/plugins/datasource/prometheus/specs/language_provider.test.ts b/public/app/plugins/datasource/prometheus/specs/language_provider.test.ts index 47102d11265..fe1679eb94b 100644 --- a/public/app/plugins/datasource/prometheus/specs/language_provider.test.ts +++ b/public/app/plugins/datasource/prometheus/specs/language_provider.test.ts @@ -6,6 +6,7 @@ import LanguageProvider from '../language_provider'; describe('Language completion provider', () => { const datasource = { metadataRequest: () => ({ data: { data: [] as any[] } }), + getTimeRange: () => ({ start: 0, end: 1 }), }; describe('empty query suggestions', () => { From 98ba3f34a6105df377aefff9836336d1122b115d Mon Sep 17 00:00:00 2001 From: kay delaney <45561153+kaydelaney@users.noreply.github.com> Date: Mon, 8 Jul 2019 16:48:47 +0100 Subject: [PATCH 5/5] Docs: Documents new features available with Loki data source in Explore (#17984) * Docs: Documents new features available with Loki data source in Explore Closes #17898 * Docs: Formats note as a quote and adds additional detail regarding live tailing --- docs/sources/features/datasources/loki.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/sources/features/datasources/loki.md b/docs/sources/features/datasources/loki.md index 11c336b7a44..07c69d375a0 100644 --- a/docs/sources/features/datasources/loki.md +++ b/docs/sources/features/datasources/loki.md @@ -92,6 +92,23 @@ Example queries: * `{name="kafka"} tsdb-ops.*io:2003` * `{instance=~"kafka-[23]",name="kafka"} kafka.server:type=ReplicaManager` +## Live Tailing + +To view your logs live as they are added, choose `Live` from the refresh dropdown, and you should see your logs be displayed in real time. + +Note that Live Tailing relies on two Websocket connections: one between the browser and the Grafana server, and another between the Grafana server and the Loki server. If you run any reverse proxies, please configure them accordingly. + + +> Note: This feature is only available in Grafana v6.3+ + +## Log Context + +When using a search expression as detailed above, you now have the ability to retrieve the context surrounding your filtered results. +By clicking the `Show Context` link on the filtered rows, you'll be able to investigate the log messages that came before and after the +log message you're interested in. + +> Note: This feature is only available in Grafana v6.3+ + ## Templating Template variables are not yet supported by Loki.