From 4803b8f3c06e130a5baa3523e1255667390d3f25 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 23 Nov 2018 17:53:16 +0100 Subject: [PATCH 1/9] Explore: Scan for older logs Sometimes log streams dont return any lines for the given range. Would be great to automate the search until some logs are found. - Allow Explore to drive TimePicker via ref - Show `Scan` link in Logs when there is no data - Click on `Scan` sets Explore into scanning state - While scanning, tell Timepicker to shift left - TimePicker change triggers new queries with shifted time range - Remember if query transaction was started via scan - keep scanning until something was found - Manual use of timepicker cancels scanning --- public/app/features/explore/Explore.tsx | 41 +++++++++++++++++++--- public/app/features/explore/Logs.tsx | 23 ++++++++++-- public/app/features/explore/TimePicker.tsx | 9 ++--- public/app/types/explore.ts | 2 ++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index d4e9b689495..d451dc6ea56 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -97,6 +97,10 @@ export class Explore extends React.PureComponent { * Local ID cache to compare requested vs selected datasource */ requestedDatasourceId: string; + /** + * Timepicker to control scanning + */ + timepickerRef: React.RefObject; constructor(props) { super(props); @@ -122,6 +126,7 @@ export class Explore extends React.PureComponent { history: [], queryTransactions: [], range: initialRange, + scanning: false, showingGraph: true, showingLogs: true, showingStartPage: false, @@ -132,6 +137,7 @@ export class Explore extends React.PureComponent { }; } this.modifiedQueries = initialQueries.slice(); + this.timepickerRef = React.createRef(); } async componentDidMount() { @@ -317,11 +323,14 @@ export class Explore extends React.PureComponent { } }; - onChangeTime = (nextRange: RawTimeRange) => { + onChangeTime = (nextRange: RawTimeRange, scanning?: boolean) => { const range: RawTimeRange = { ...nextRange, }; - this.setState({ range }, () => this.onSubmit()); + if (this.state.scanning && !scanning) { + this.stopScanOlder(); + } + this.setState({ range, scanning }, () => this.onSubmit()); }; onClickClear = () => { @@ -496,6 +505,18 @@ export class Explore extends React.PureComponent { ); }; + onStartScanOlder = () => { + this.setState({ scanning: true }, this.scanOlder); + }; + + scanOlder = () => { + this.timepickerRef.current.move(-1, true); + }; + + stopScanOlder = () => { + // Stop ongoing scan transactions + }; + onSubmit = () => { const { showingLogs, showingGraph, showingTable, supportsGraph, supportsLogs, supportsTable } = this.state; // Keep table queries first since they need to return quickly @@ -563,6 +584,7 @@ export class Explore extends React.PureComponent { done: false, latency: 0, options: queryOptions, + scanning: this.state.scanning, }; // Using updater style because we might be modifying queryTransactions in quick succession @@ -599,7 +621,7 @@ export class Explore extends React.PureComponent { } this.setState(state => { - const { history, queryTransactions } = state; + const { history, queryTransactions, scanning } = state; // Transaction might have been discarded const transaction = queryTransactions.find(qt => qt.id === transactionId); @@ -629,6 +651,14 @@ export class Explore extends React.PureComponent { const nextHistory = updateHistory(history, datasourceId, queries); + if (_.size(result) === 0 && scanning) { + // Keep scanning if this was the last scanning transaction + const other = nextQueryTransactions.find(qt => qt.scanning && !qt.done); + if (!other) { + setTimeout(this.scanOlder, 1000); + } + } + return { history: nextHistory, queryTransactions: nextQueryTransactions, @@ -740,6 +770,7 @@ export class Explore extends React.PureComponent { initialQueries, queryTransactions, range, + scanning, showingGraph, showingLogs, showingStartPage, @@ -822,7 +853,7 @@ export class Explore extends React.PureComponent { ) : null} - +
- {!loading && !hasData && 'No data was returned.'} + {!loading && + !hasData && ( +
+ No logs found. + {scanning ? ( + 'Scanning...' + ) : ( + + Scan for older logs + + )} +
+ )} ); } diff --git a/public/app/features/explore/TimePicker.tsx b/public/app/features/explore/TimePicker.tsx index a3578263cea..ebfb23087d2 100644 --- a/public/app/features/explore/TimePicker.tsx +++ b/public/app/features/explore/TimePicker.tsx @@ -35,7 +35,7 @@ interface TimePickerProps { isOpen?: boolean; isUtc?: boolean; range?: RawTimeRange; - onChangeTime?: (Range) => void; + onChangeTime?: (range: RawTimeRange, scanning?: boolean) => void; } interface TimePickerState { @@ -92,12 +92,13 @@ export default class TimePicker extends PureComponent { - onChangeTime(nextRange); + onChangeTime(nextRange, scanning); } ); } diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index f80a485fc29..3aef458bd54 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -140,6 +140,7 @@ export interface QueryTransaction { result?: any; // Table model / Timeseries[] / Logs resultType: ResultType; rowIndex: number; + scanning?: boolean; } export interface TextMatch { @@ -162,6 +163,7 @@ export interface ExploreState { initialQueries: DataQuery[]; queryTransactions: QueryTransaction[]; range: RawTimeRange; + scanning?: boolean; showingGraph: boolean; showingLogs: boolean; showingStartPage?: boolean; From 593cc38cfc36709839878d8ce212bd9158d404f6 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 27 Nov 2018 16:35:37 +0100 Subject: [PATCH 2/9] Added stop scan button --- public/app/features/explore/Explore.tsx | 34 ++++++++++++------ public/app/features/explore/Logs.tsx | 41 +++++++++++++++------- public/app/features/explore/TimePicker.tsx | 4 ++- public/app/types/explore.ts | 1 + public/sass/pages/_explore.scss | 6 ++++ 5 files changed, 62 insertions(+), 24 deletions(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index d451dc6ea56..44380877c34 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -97,6 +97,7 @@ export class Explore extends React.PureComponent { * Local ID cache to compare requested vs selected datasource */ requestedDatasourceId: string; + scanTimer: NodeJS.Timer; /** * Timepicker to control scanning */ @@ -170,6 +171,10 @@ export class Explore extends React.PureComponent { } } + componentWillUnmount() { + clearTimeout(this.scanTimer); + } + async setDatasource(datasource: any, origin?: DataSource) { const supportsGraph = datasource.meta.metrics; const supportsLogs = datasource.meta.logs; @@ -328,7 +333,7 @@ export class Explore extends React.PureComponent { ...nextRange, }; if (this.state.scanning && !scanning) { - this.stopScanOlder(); + this.onStopScanning(); } this.setState({ range, scanning }, () => this.onSubmit()); }; @@ -505,16 +510,22 @@ export class Explore extends React.PureComponent { ); }; - onStartScanOlder = () => { - this.setState({ scanning: true }, this.scanOlder); + onStartScanning = () => { + this.setState({ scanning: true }, this.scanPreviousRange); }; - scanOlder = () => { - this.timepickerRef.current.move(-1, true); + scanPreviousRange = () => { + const scanRange = this.timepickerRef.current.move(-1, true); + this.setState({ scanRange }); }; - stopScanOlder = () => { - // Stop ongoing scan transactions + onStopScanning = () => { + clearTimeout(this.scanTimer); + this.setState(state => { + const { queryTransactions } = state; + const nextQueryTransactions = queryTransactions.filter(qt => qt.scanning && !qt.done); + return { queryTransactions: nextQueryTransactions, scanning: false, scanRange: undefined }; + }); }; onSubmit = () => { @@ -651,11 +662,11 @@ export class Explore extends React.PureComponent { const nextHistory = updateHistory(history, datasourceId, queries); + // Keep scanning for results if this was the last scanning transaction if (_.size(result) === 0 && scanning) { - // Keep scanning if this was the last scanning transaction const other = nextQueryTransactions.find(qt => qt.scanning && !qt.done); if (!other) { - setTimeout(this.scanOlder, 1000); + this.scanTimer = setTimeout(this.scanPreviousRange, 1000); } } @@ -771,6 +782,7 @@ export class Explore extends React.PureComponent { queryTransactions, range, scanning, + scanRange, showingGraph, showingLogs, showingStartPage, @@ -929,9 +941,11 @@ export class Explore extends React.PureComponent { loading={logsLoading} position={position} onChangeTime={this.onChangeTime} - onStartScanOlder={this.onStartScanOlder} + onStartScanning={this.onStartScanning} + onStopScanning={this.onStopScanning} range={range} scanning={scanning} + scanRange={scanRange} /> )} diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 37feb719f63..58965df4514 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -1,6 +1,7 @@ import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; +import * as rangeUtil from 'app/core/utils/rangeutil'; import { RawTimeRange } from 'app/types/series'; import { LogsDedupStrategy, LogsModel, dedupLogRows, filterLogLevels, LogLevel } from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; @@ -29,8 +30,10 @@ interface LogsProps { position: string; range?: RawTimeRange; scanning?: boolean; + scanRange?: RawTimeRange; onChangeTime?: (range: RawTimeRange) => void; - onStartScanOlder?: () => void; + onStartScanning?: () => void; + onStopScanning?: () => void; } interface LogsState { @@ -85,13 +88,18 @@ export default class Logs extends PureComponent { this.setState({ hiddenLogLevels }); }; - onClickScanOlder = (event: React.SyntheticEvent) => { + onClickScan = (event: React.SyntheticEvent) => { event.preventDefault(); - this.props.onStartScanOlder(); + this.props.onStartScanning(); + }; + + onClickStopScan = (event: React.SyntheticEvent) => { + event.preventDefault(); + this.props.onStopScanning(); }; render() { - const { className = '', data, loading = false, position, range, scanning } = this.props; + const { className = '', data, loading = false, position, range, scanning, scanRange } = this.props; const { dedup, hiddenLogLevels, showLabels, showLocalTime, showUtc } = this.state; const hasData = data && data.rows && data.rows.length > 0; const filteredData = filterLogLevels(data, hiddenLogLevels); @@ -118,6 +126,7 @@ export default class Logs extends PureComponent { const logEntriesStyle = { gridTemplateColumns: cssColumnSizes.join(' '), }; + const scanText = scanRange ? `Scanning ${rangeUtil.describeTimeRange(scanRange)}` : 'Scanning...'; return (
@@ -208,18 +217,24 @@ export default class Logs extends PureComponent { ))}
{!loading && - !hasData && ( -
+ !hasData && + !scanning && ( +
No logs found. - {scanning ? ( - 'Scanning...' - ) : ( - - Scan for older logs - - )} + + Scan for older logs +
)} + + {scanning && ( +
+ {scanText} + + Stop scan + +
+ )}
); } diff --git a/public/app/features/explore/TimePicker.tsx b/public/app/features/explore/TimePicker.tsx index ebfb23087d2..47c52b07292 100644 --- a/public/app/features/explore/TimePicker.tsx +++ b/public/app/features/explore/TimePicker.tsx @@ -92,7 +92,7 @@ export default class TimePicker extends PureComponent { diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index 3aef458bd54..d9ace7b74c0 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -164,6 +164,7 @@ export interface ExploreState { queryTransactions: QueryTransaction[]; range: RawTimeRange; scanning?: boolean; + scanRange?: RawTimeRange; showingGraph: boolean; showingLogs: boolean; showingStartPage?: boolean; diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 23c6fbf0916..5c2848d018b 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -267,6 +267,12 @@ } } + .logs-nodata { + > * { + margin-left: 0.5em; + } + } + .logs-meta { flex: 1; color: $text-color-weak; From 2faf8c722f7b77adc291365433c90ac51388ec36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 28 Nov 2018 09:20:49 +0100 Subject: [PATCH 3/9] Fix elastic ng-inject (build issue) (#14195) fix elastic ng-inject issue in query editor --- .../datasource/elasticsearch/bucket_agg.ts | 31 +++++++++--------- .../datasource/elasticsearch/metric_agg.ts | 32 +++++++++---------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/bucket_agg.ts b/public/app/plugins/datasource/elasticsearch/bucket_agg.ts index cacf86201fe..8701b2bf335 100644 --- a/public/app/plugins/datasource/elasticsearch/bucket_agg.ts +++ b/public/app/plugins/datasource/elasticsearch/bucket_agg.ts @@ -2,22 +2,8 @@ import coreModule from 'app/core/core_module'; import _ from 'lodash'; import * as queryDef from './query_def'; -export function elasticBucketAgg() { - return { - templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html', - controller: 'ElasticBucketAggCtrl', - restrict: 'E', - scope: { - target: '=', - index: '=', - onChange: '&', - getFields: '&', - }, - }; -} - export class ElasticBucketAggCtrl { - /** @nginject */ + /** @ngInject */ constructor($scope, uiSegmentSrv, $q, $rootScope) { const bucketAggs = $scope.target.bucketAggs; @@ -226,5 +212,18 @@ export class ElasticBucketAggCtrl { } } +export function elasticBucketAgg() { + return { + templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html', + controller: ElasticBucketAggCtrl, + restrict: 'E', + scope: { + target: '=', + index: '=', + onChange: '&', + getFields: '&', + }, + }; +} + coreModule.directive('elasticBucketAgg', elasticBucketAgg); -coreModule.controller('ElasticBucketAggCtrl', ElasticBucketAggCtrl); diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.ts b/public/app/plugins/datasource/elasticsearch/metric_agg.ts index 56f874d90b9..cae5be45720 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.ts +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.ts @@ -2,22 +2,8 @@ import coreModule from 'app/core/core_module'; import _ from 'lodash'; import * as queryDef from './query_def'; -export function elasticMetricAgg() { - return { - templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/metric_agg.html', - controller: 'ElasticMetricAggCtrl', - restrict: 'E', - scope: { - target: '=', - index: '=', - onChange: '&', - getFields: '&', - esVersion: '=', - }, - }; -} - export class ElasticMetricAggCtrl { + /** @ngInject */ constructor($scope, uiSegmentSrv, $q, $rootScope) { const metricAggs = $scope.target.metrics; $scope.metricAggTypes = queryDef.getMetricAggTypes($scope.esVersion); @@ -209,5 +195,19 @@ export class ElasticMetricAggCtrl { } } +export function elasticMetricAgg() { + return { + templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/metric_agg.html', + controller: ElasticMetricAggCtrl, + restrict: 'E', + scope: { + target: '=', + index: '=', + onChange: '&', + getFields: '&', + esVersion: '=', + }, + }; +} + coreModule.directive('elasticMetricAgg', elasticMetricAgg); -coreModule.controller('ElasticMetricAggCtrl', ElasticMetricAggCtrl); From b3e6da0cbd5dc1478f51472c1df57eba550af422 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 28 Nov 2018 00:24:59 -0800 Subject: [PATCH 4/9] check for null with toLocalString (#14208) --- public/app/core/utils/kbn.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/app/core/utils/kbn.ts b/public/app/core/utils/kbn.ts index 5d417d24169..9caa5bf1a54 100644 --- a/public/app/core/utils/kbn.ts +++ b/public/app/core/utils/kbn.ts @@ -428,10 +428,16 @@ kbn.valueFormats.hex0x = (value, decimals) => { }; kbn.valueFormats.sci = (value, decimals) => { + if (value == null) { + return ''; + } return value.toExponential(decimals); }; kbn.valueFormats.locale = (value, decimals) => { + if (value == null) { + return ''; + } return value.toLocaleString(undefined, { maximumFractionDigits: decimals }); }; From ce9e1a8f385814b53444e7c40bec2cca25baa0e8 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Tue, 27 Nov 2018 16:55:59 +0100 Subject: [PATCH 5/9] build: explaining the linux build. --- scripts/build/build-all.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build/build-all.sh b/scripts/build/build-all.sh index be0b297527b..3013452a279 100755 --- a/scripts/build/build-all.sh +++ b/scripts/build/build-all.sh @@ -35,6 +35,8 @@ go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build go run build.go -goos windows -cc ${CCWIN64} ${OPT} build + +# Do not remove CC from the linux build, its there for compatibility with Centos6 CC=${CCX64} go run build.go ${OPT} build yarn install --pure-lockfile --no-progress From 9c316b55e9415d07ec2f937024ce02456c589ec8 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 28 Nov 2018 14:03:47 +0100 Subject: [PATCH 6/9] Logging: fix query parsing for selectors with multiple labels - simplify selector parsing - added tests --- .../datasource/logging/datasource.test.ts | 7 +++++++ .../plugins/datasource/logging/datasource.ts | 21 +++++-------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/public/app/plugins/datasource/logging/datasource.test.ts b/public/app/plugins/datasource/logging/datasource.test.ts index 212d352dfca..2e2adb144ef 100644 --- a/public/app/plugins/datasource/logging/datasource.test.ts +++ b/public/app/plugins/datasource/logging/datasource.test.ts @@ -35,4 +35,11 @@ describe('parseQuery', () => { regexp: 'x|y', }); }); + + it('returns query for selector with two labels', () => { + expect(parseQuery('{foo="bar", baz="42"}')).toEqual({ + query: '{foo="bar", baz="42"}', + regexp: '', + }); + }); }); diff --git a/public/app/plugins/datasource/logging/datasource.ts b/public/app/plugins/datasource/logging/datasource.ts index 494dcd78d6c..e36da73cd66 100644 --- a/public/app/plugins/datasource/logging/datasource.ts +++ b/public/app/plugins/datasource/logging/datasource.ts @@ -16,26 +16,15 @@ const DEFAULT_QUERY_PARAMS = { query: '', }; -const QUERY_REGEXP = /({\w+="[^"]+"})?\s*(\w[^{]+)?\s*({\w+="[^"]+"})?/; +const selectorRegexp = /{[^{]*}/g; export function parseQuery(input: string) { - const match = input.match(QUERY_REGEXP); + const match = input.match(selectorRegexp); let query = ''; - let regexp = ''; + let regexp = input; if (match) { - if (match[1]) { - query = match[1]; - } - if (match[2]) { - regexp = match[2].trim(); - } - if (match[3]) { - if (match[1]) { - query = `${match[1].slice(0, -1)},${match[3].slice(1)}`; - } else { - query = match[3]; - } - } + query = match[0]; + regexp = input.replace(selectorRegexp, '').trim(); } return { query, regexp }; From 804bd822d0fddcf21610a0e83ec6d6a172aa236b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Omar=20Alejandro=20Gonz=C3=A1lez=20Rojina?= Date: Wed, 28 Nov 2018 11:17:36 -0600 Subject: [PATCH 7/9] Update export_import.md Grafana v5.3.4 shows a new checkbox in the export modal "Export for sharing externally". If the checkbox is not checked then the "__inputs" section wont be included into the exported JSON file, would be great to add that note into the documentation for others to avoid confusions. --- docs/sources/reference/export_import.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sources/reference/export_import.md b/docs/sources/reference/export_import.md index 31f32d890f6..501e9aca62b 100644 --- a/docs/sources/reference/export_import.md +++ b/docs/sources/reference/export_import.md @@ -107,3 +107,5 @@ it as usual and then update the data source option in the metrics tab so that th data source. Another alternative is to open the json file in a a text editor and update the data source properties to value that matches a name of your data source. +## Note +In Grafana v5.3.4+ the export modal has new checkbox for sharing for external use (other instances). If the checkbox is not checked then the `__inputs` section will not be included in the exported JSON file. From d86ba20d10d955cf02c3c7d8c3dfea36aca30cd3 Mon Sep 17 00:00:00 2001 From: Florian Zicklam Date: Thu, 29 Nov 2018 09:21:06 +0100 Subject: [PATCH 8/9] removed extra whitespace removed extra whitespace --- conf/defaults.ini | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 306c625d980..a9aa2239b16 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -34,7 +34,7 @@ protocol = http # The ip address to bind to, empty will bind to all interfaces http_addr = -# The http port to use +# The http port to use http_port = 3000 # The public facing domain name used to access grafana from a browser @@ -166,7 +166,7 @@ google_tag_manager_id = # default admin user, created on startup admin_user = admin -# default admin password, can be changed before first start of grafana, or in profile settings +# default admin password, can be changed before first start of grafana, or in profile settings admin_password = admin # used for signing @@ -372,7 +372,7 @@ templates_pattern = emails/*.html #################################### Logging ########################## [log] -# Either "console", "file", "syslog". Default is console and file +# Either "console", "file", "syslog". Default is console and file # Use space to separate multiple modes, e.g. "console file" mode = console file @@ -565,4 +565,3 @@ enable_alpha = false [enterprise] license_path = - From 3000818ab38b706bc651c3b0298c103d6071e9d4 Mon Sep 17 00:00:00 2001 From: Florian Zicklam Date: Thu, 29 Nov 2018 09:30:03 +0100 Subject: [PATCH 9/9] added google_tag_manager_id from defaults.ini --- conf/sample.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/sample.ini b/conf/sample.ini index c6b716a731d..3c61b2b61d1 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -145,6 +145,9 @@ log_queries = # Google Analytics universal tracking code, only enabled if you specify an id here ;google_analytics_ua_id = +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + #################################### Security #################################### [security] # default admin user, created on startup