From 982548417681d8cf92b40eb052ede7e8b1cc7bd4 Mon Sep 17 00:00:00 2001 From: Mattia Rossi Date: Sat, 24 Nov 2018 10:48:29 -0500 Subject: [PATCH 01/39] Add support for Offset in elasticsearch datasource, date_histogram aggregation, fixes grafana #12653 --- .../datasource/elasticsearch/partials/bucket_agg.html | 10 ++++++++++ .../plugins/datasource/elasticsearch/query_builder.ts | 1 + 2 files changed, 11 insertions(+) diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index d0cf23a3d93..a6f68685681 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -70,6 +70,16 @@ +
+ + +
+
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index efdeee68370..7d9490521e3 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -71,6 +71,7 @@ export class ElasticQueryBuilder { esAgg.min_doc_count = settings.min_doc_count || 0; esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' }; esAgg.format = 'epoch_millis'; + esAgg.offset = settings.offset || '0'; if (esAgg.interval === 'auto') { esAgg.interval = '$__interval'; From f6fc7f7b64f7f51e519c722459f849765f6d55fb Mon Sep 17 00:00:00 2001 From: Mattia Rossi Date: Mon, 26 Nov 2018 08:58:25 -0500 Subject: [PATCH 02/39] Requested Backend changes, added details to popover description for the offset field --- pkg/tsdb/elasticsearch/client/models.go | 1 + pkg/tsdb/elasticsearch/time_series_query.go | 3 +++ .../datasource/elasticsearch/partials/bucket_agg.html | 3 ++- public/app/plugins/datasource/elasticsearch/query_builder.ts | 5 ++++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index a0d257d01a6..5307dc34b27 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -240,6 +240,7 @@ type DateHistogramAgg struct { Missing *string `json:"missing,omitempty"` ExtendedBounds *ExtendedBounds `json:"extended_bounds"` Format string `json:"format"` + Offset string `json:"offset,omitempty"` } // FiltersAggregation represents a filters aggregation diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 869e23e21ce..3e43a26089a 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -134,6 +134,9 @@ func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFro a.Interval = "$__interval" } + if bucketAgg.Settings.Get("offset").MustString("") != "" { + a.Offset = bucketAgg.Settings.Get("offset").MustString("") + } if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { a.Missing = &missing } diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index a6f68685681..9d6b8196e8e 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -74,7 +74,8 @@ diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index 7d9490521e3..c505d402874 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -71,7 +71,10 @@ export class ElasticQueryBuilder { esAgg.min_doc_count = settings.min_doc_count || 0; esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' }; esAgg.format = 'epoch_millis'; - esAgg.offset = settings.offset || '0'; + + if (settings.offset !== '') { + esAgg.offset = settings.offset; + } if (esAgg.interval === 'auto') { esAgg.interval = '$__interval'; From ce74b1ddc23fd3ab94dbd58064c16f85f49119a9 Mon Sep 17 00:00:00 2001 From: Mattia Rossi Date: Tue, 27 Nov 2018 03:42:20 -0500 Subject: [PATCH 03/39] Requested Backend changes, removed link in popover description for the offset field --- pkg/tsdb/elasticsearch/time_series_query.go | 5 +++-- .../datasource/elasticsearch/partials/bucket_agg.html | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 3e43a26089a..a25c3cf693d 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -134,9 +134,10 @@ func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFro a.Interval = "$__interval" } - if bucketAgg.Settings.Get("offset").MustString("") != "" { - a.Offset = bucketAgg.Settings.Get("offset").MustString("") + if offset, err := bucketAgg.Settings.Get("offset").String(); err == nil { + a.Offset = offset } + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { a.Missing = &missing } diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index 9d6b8196e8e..b17f03bced3 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -74,8 +74,7 @@ From 4803b8f3c06e130a5baa3523e1255667390d3f25 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 23 Nov 2018 17:53:16 +0100 Subject: [PATCH 04/39] Explore: Scan for older logs Sometimes log streams dont return any lines for the given range. Would be great to automate the search until some logs are found. - Allow Explore to drive TimePicker via ref - Show `Scan` link in Logs when there is no data - Click on `Scan` sets Explore into scanning state - While scanning, tell Timepicker to shift left - TimePicker change triggers new queries with shifted time range - Remember if query transaction was started via scan - keep scanning until something was found - Manual use of timepicker cancels scanning --- public/app/features/explore/Explore.tsx | 41 +++++++++++++++++++--- public/app/features/explore/Logs.tsx | 23 ++++++++++-- public/app/features/explore/TimePicker.tsx | 9 ++--- public/app/types/explore.ts | 2 ++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index d4e9b689495..d451dc6ea56 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -97,6 +97,10 @@ export class Explore extends React.PureComponent { * Local ID cache to compare requested vs selected datasource */ requestedDatasourceId: string; + /** + * Timepicker to control scanning + */ + timepickerRef: React.RefObject; constructor(props) { super(props); @@ -122,6 +126,7 @@ export class Explore extends React.PureComponent { history: [], queryTransactions: [], range: initialRange, + scanning: false, showingGraph: true, showingLogs: true, showingStartPage: false, @@ -132,6 +137,7 @@ export class Explore extends React.PureComponent { }; } this.modifiedQueries = initialQueries.slice(); + this.timepickerRef = React.createRef(); } async componentDidMount() { @@ -317,11 +323,14 @@ export class Explore extends React.PureComponent { } }; - onChangeTime = (nextRange: RawTimeRange) => { + onChangeTime = (nextRange: RawTimeRange, scanning?: boolean) => { const range: RawTimeRange = { ...nextRange, }; - this.setState({ range }, () => this.onSubmit()); + if (this.state.scanning && !scanning) { + this.stopScanOlder(); + } + this.setState({ range, scanning }, () => this.onSubmit()); }; onClickClear = () => { @@ -496,6 +505,18 @@ export class Explore extends React.PureComponent { ); }; + onStartScanOlder = () => { + this.setState({ scanning: true }, this.scanOlder); + }; + + scanOlder = () => { + this.timepickerRef.current.move(-1, true); + }; + + stopScanOlder = () => { + // Stop ongoing scan transactions + }; + onSubmit = () => { const { showingLogs, showingGraph, showingTable, supportsGraph, supportsLogs, supportsTable } = this.state; // Keep table queries first since they need to return quickly @@ -563,6 +584,7 @@ export class Explore extends React.PureComponent { done: false, latency: 0, options: queryOptions, + scanning: this.state.scanning, }; // Using updater style because we might be modifying queryTransactions in quick succession @@ -599,7 +621,7 @@ export class Explore extends React.PureComponent { } this.setState(state => { - const { history, queryTransactions } = state; + const { history, queryTransactions, scanning } = state; // Transaction might have been discarded const transaction = queryTransactions.find(qt => qt.id === transactionId); @@ -629,6 +651,14 @@ export class Explore extends React.PureComponent { const nextHistory = updateHistory(history, datasourceId, queries); + if (_.size(result) === 0 && scanning) { + // Keep scanning if this was the last scanning transaction + const other = nextQueryTransactions.find(qt => qt.scanning && !qt.done); + if (!other) { + setTimeout(this.scanOlder, 1000); + } + } + return { history: nextHistory, queryTransactions: nextQueryTransactions, @@ -740,6 +770,7 @@ export class Explore extends React.PureComponent { initialQueries, queryTransactions, range, + scanning, showingGraph, showingLogs, showingStartPage, @@ -822,7 +853,7 @@ export class Explore extends React.PureComponent {
) : null} - +
- {!loading && !hasData && 'No data was returned.'} + {!loading && + !hasData && ( +
+ No logs found. + {scanning ? ( + 'Scanning...' + ) : ( + + Scan for older logs + + )} +
+ )} ); } diff --git a/public/app/features/explore/TimePicker.tsx b/public/app/features/explore/TimePicker.tsx index a3578263cea..ebfb23087d2 100644 --- a/public/app/features/explore/TimePicker.tsx +++ b/public/app/features/explore/TimePicker.tsx @@ -35,7 +35,7 @@ interface TimePickerProps { isOpen?: boolean; isUtc?: boolean; range?: RawTimeRange; - onChangeTime?: (Range) => void; + onChangeTime?: (range: RawTimeRange, scanning?: boolean) => void; } interface TimePickerState { @@ -92,12 +92,13 @@ export default class TimePicker extends PureComponent { - onChangeTime(nextRange); + onChangeTime(nextRange, scanning); } ); } diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index f80a485fc29..3aef458bd54 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -140,6 +140,7 @@ export interface QueryTransaction { result?: any; // Table model / Timeseries[] / Logs resultType: ResultType; rowIndex: number; + scanning?: boolean; } export interface TextMatch { @@ -162,6 +163,7 @@ export interface ExploreState { initialQueries: DataQuery[]; queryTransactions: QueryTransaction[]; range: RawTimeRange; + scanning?: boolean; showingGraph: boolean; showingLogs: boolean; showingStartPage?: boolean; From 593cc38cfc36709839878d8ce212bd9158d404f6 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 27 Nov 2018 16:35:37 +0100 Subject: [PATCH 05/39] Added stop scan button --- public/app/features/explore/Explore.tsx | 34 ++++++++++++------ public/app/features/explore/Logs.tsx | 41 +++++++++++++++------- public/app/features/explore/TimePicker.tsx | 4 ++- public/app/types/explore.ts | 1 + public/sass/pages/_explore.scss | 6 ++++ 5 files changed, 62 insertions(+), 24 deletions(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index d451dc6ea56..44380877c34 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -97,6 +97,7 @@ export class Explore extends React.PureComponent { * Local ID cache to compare requested vs selected datasource */ requestedDatasourceId: string; + scanTimer: NodeJS.Timer; /** * Timepicker to control scanning */ @@ -170,6 +171,10 @@ export class Explore extends React.PureComponent { } } + componentWillUnmount() { + clearTimeout(this.scanTimer); + } + async setDatasource(datasource: any, origin?: DataSource) { const supportsGraph = datasource.meta.metrics; const supportsLogs = datasource.meta.logs; @@ -328,7 +333,7 @@ export class Explore extends React.PureComponent { ...nextRange, }; if (this.state.scanning && !scanning) { - this.stopScanOlder(); + this.onStopScanning(); } this.setState({ range, scanning }, () => this.onSubmit()); }; @@ -505,16 +510,22 @@ export class Explore extends React.PureComponent { ); }; - onStartScanOlder = () => { - this.setState({ scanning: true }, this.scanOlder); + onStartScanning = () => { + this.setState({ scanning: true }, this.scanPreviousRange); }; - scanOlder = () => { - this.timepickerRef.current.move(-1, true); + scanPreviousRange = () => { + const scanRange = this.timepickerRef.current.move(-1, true); + this.setState({ scanRange }); }; - stopScanOlder = () => { - // Stop ongoing scan transactions + onStopScanning = () => { + clearTimeout(this.scanTimer); + this.setState(state => { + const { queryTransactions } = state; + const nextQueryTransactions = queryTransactions.filter(qt => qt.scanning && !qt.done); + return { queryTransactions: nextQueryTransactions, scanning: false, scanRange: undefined }; + }); }; onSubmit = () => { @@ -651,11 +662,11 @@ export class Explore extends React.PureComponent { const nextHistory = updateHistory(history, datasourceId, queries); + // Keep scanning for results if this was the last scanning transaction if (_.size(result) === 0 && scanning) { - // Keep scanning if this was the last scanning transaction const other = nextQueryTransactions.find(qt => qt.scanning && !qt.done); if (!other) { - setTimeout(this.scanOlder, 1000); + this.scanTimer = setTimeout(this.scanPreviousRange, 1000); } } @@ -771,6 +782,7 @@ export class Explore extends React.PureComponent { queryTransactions, range, scanning, + scanRange, showingGraph, showingLogs, showingStartPage, @@ -929,9 +941,11 @@ export class Explore extends React.PureComponent { loading={logsLoading} position={position} onChangeTime={this.onChangeTime} - onStartScanOlder={this.onStartScanOlder} + onStartScanning={this.onStartScanning} + onStopScanning={this.onStopScanning} range={range} scanning={scanning} + scanRange={scanRange} /> )} diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 37feb719f63..58965df4514 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -1,6 +1,7 @@ import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; +import * as rangeUtil from 'app/core/utils/rangeutil'; import { RawTimeRange } from 'app/types/series'; import { LogsDedupStrategy, LogsModel, dedupLogRows, filterLogLevels, LogLevel } from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; @@ -29,8 +30,10 @@ interface LogsProps { position: string; range?: RawTimeRange; scanning?: boolean; + scanRange?: RawTimeRange; onChangeTime?: (range: RawTimeRange) => void; - onStartScanOlder?: () => void; + onStartScanning?: () => void; + onStopScanning?: () => void; } interface LogsState { @@ -85,13 +88,18 @@ export default class Logs extends PureComponent { this.setState({ hiddenLogLevels }); }; - onClickScanOlder = (event: React.SyntheticEvent) => { + onClickScan = (event: React.SyntheticEvent) => { event.preventDefault(); - this.props.onStartScanOlder(); + this.props.onStartScanning(); + }; + + onClickStopScan = (event: React.SyntheticEvent) => { + event.preventDefault(); + this.props.onStopScanning(); }; render() { - const { className = '', data, loading = false, position, range, scanning } = this.props; + const { className = '', data, loading = false, position, range, scanning, scanRange } = this.props; const { dedup, hiddenLogLevels, showLabels, showLocalTime, showUtc } = this.state; const hasData = data && data.rows && data.rows.length > 0; const filteredData = filterLogLevels(data, hiddenLogLevels); @@ -118,6 +126,7 @@ export default class Logs extends PureComponent { const logEntriesStyle = { gridTemplateColumns: cssColumnSizes.join(' '), }; + const scanText = scanRange ? `Scanning ${rangeUtil.describeTimeRange(scanRange)}` : 'Scanning...'; return (
@@ -208,18 +217,24 @@ export default class Logs extends PureComponent { ))}
{!loading && - !hasData && ( -
+ !hasData && + !scanning && ( +
No logs found. - {scanning ? ( - 'Scanning...' - ) : ( - - Scan for older logs - - )} + + Scan for older logs +
)} + + {scanning && ( +
+ {scanText} + + Stop scan + +
+ )}
); } diff --git a/public/app/features/explore/TimePicker.tsx b/public/app/features/explore/TimePicker.tsx index ebfb23087d2..47c52b07292 100644 --- a/public/app/features/explore/TimePicker.tsx +++ b/public/app/features/explore/TimePicker.tsx @@ -92,7 +92,7 @@ export default class TimePicker extends PureComponent { diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index 3aef458bd54..d9ace7b74c0 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -164,6 +164,7 @@ export interface ExploreState { queryTransactions: QueryTransaction[]; range: RawTimeRange; scanning?: boolean; + scanRange?: RawTimeRange; showingGraph: boolean; showingLogs: boolean; showingStartPage?: boolean; diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 23c6fbf0916..5c2848d018b 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -267,6 +267,12 @@ } } + .logs-nodata { + > * { + margin-left: 0.5em; + } + } + .logs-meta { flex: 1; color: $text-color-weak; From 2faf8c722f7b77adc291365433c90ac51388ec36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 28 Nov 2018 09:20:49 +0100 Subject: [PATCH 06/39] Fix elastic ng-inject (build issue) (#14195) fix elastic ng-inject issue in query editor --- .../datasource/elasticsearch/bucket_agg.ts | 31 +++++++++--------- .../datasource/elasticsearch/metric_agg.ts | 32 +++++++++---------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/bucket_agg.ts b/public/app/plugins/datasource/elasticsearch/bucket_agg.ts index cacf86201fe..8701b2bf335 100644 --- a/public/app/plugins/datasource/elasticsearch/bucket_agg.ts +++ b/public/app/plugins/datasource/elasticsearch/bucket_agg.ts @@ -2,22 +2,8 @@ import coreModule from 'app/core/core_module'; import _ from 'lodash'; import * as queryDef from './query_def'; -export function elasticBucketAgg() { - return { - templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html', - controller: 'ElasticBucketAggCtrl', - restrict: 'E', - scope: { - target: '=', - index: '=', - onChange: '&', - getFields: '&', - }, - }; -} - export class ElasticBucketAggCtrl { - /** @nginject */ + /** @ngInject */ constructor($scope, uiSegmentSrv, $q, $rootScope) { const bucketAggs = $scope.target.bucketAggs; @@ -226,5 +212,18 @@ export class ElasticBucketAggCtrl { } } +export function elasticBucketAgg() { + return { + templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html', + controller: ElasticBucketAggCtrl, + restrict: 'E', + scope: { + target: '=', + index: '=', + onChange: '&', + getFields: '&', + }, + }; +} + coreModule.directive('elasticBucketAgg', elasticBucketAgg); -coreModule.controller('ElasticBucketAggCtrl', ElasticBucketAggCtrl); diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.ts b/public/app/plugins/datasource/elasticsearch/metric_agg.ts index 56f874d90b9..cae5be45720 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.ts +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.ts @@ -2,22 +2,8 @@ import coreModule from 'app/core/core_module'; import _ from 'lodash'; import * as queryDef from './query_def'; -export function elasticMetricAgg() { - return { - templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/metric_agg.html', - controller: 'ElasticMetricAggCtrl', - restrict: 'E', - scope: { - target: '=', - index: '=', - onChange: '&', - getFields: '&', - esVersion: '=', - }, - }; -} - export class ElasticMetricAggCtrl { + /** @ngInject */ constructor($scope, uiSegmentSrv, $q, $rootScope) { const metricAggs = $scope.target.metrics; $scope.metricAggTypes = queryDef.getMetricAggTypes($scope.esVersion); @@ -209,5 +195,19 @@ export class ElasticMetricAggCtrl { } } +export function elasticMetricAgg() { + return { + templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/metric_agg.html', + controller: ElasticMetricAggCtrl, + restrict: 'E', + scope: { + target: '=', + index: '=', + onChange: '&', + getFields: '&', + esVersion: '=', + }, + }; +} + coreModule.directive('elasticMetricAgg', elasticMetricAgg); -coreModule.controller('ElasticMetricAggCtrl', ElasticMetricAggCtrl); From b3e6da0cbd5dc1478f51472c1df57eba550af422 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 28 Nov 2018 00:24:59 -0800 Subject: [PATCH 07/39] check for null with toLocalString (#14208) --- public/app/core/utils/kbn.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/app/core/utils/kbn.ts b/public/app/core/utils/kbn.ts index 5d417d24169..9caa5bf1a54 100644 --- a/public/app/core/utils/kbn.ts +++ b/public/app/core/utils/kbn.ts @@ -428,10 +428,16 @@ kbn.valueFormats.hex0x = (value, decimals) => { }; kbn.valueFormats.sci = (value, decimals) => { + if (value == null) { + return ''; + } return value.toExponential(decimals); }; kbn.valueFormats.locale = (value, decimals) => { + if (value == null) { + return ''; + } return value.toLocaleString(undefined, { maximumFractionDigits: decimals }); }; From ce9e1a8f385814b53444e7c40bec2cca25baa0e8 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Tue, 27 Nov 2018 16:55:59 +0100 Subject: [PATCH 08/39] build: explaining the linux build. --- scripts/build/build-all.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build/build-all.sh b/scripts/build/build-all.sh index be0b297527b..3013452a279 100755 --- a/scripts/build/build-all.sh +++ b/scripts/build/build-all.sh @@ -35,6 +35,8 @@ go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build go run build.go -goos windows -cc ${CCWIN64} ${OPT} build + +# Do not remove CC from the linux build, its there for compatibility with Centos6 CC=${CCX64} go run build.go ${OPT} build yarn install --pure-lockfile --no-progress From 9c316b55e9415d07ec2f937024ce02456c589ec8 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 28 Nov 2018 14:03:47 +0100 Subject: [PATCH 09/39] Logging: fix query parsing for selectors with multiple labels - simplify selector parsing - added tests --- .../datasource/logging/datasource.test.ts | 7 +++++++ .../plugins/datasource/logging/datasource.ts | 21 +++++-------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/public/app/plugins/datasource/logging/datasource.test.ts b/public/app/plugins/datasource/logging/datasource.test.ts index 212d352dfca..2e2adb144ef 100644 --- a/public/app/plugins/datasource/logging/datasource.test.ts +++ b/public/app/plugins/datasource/logging/datasource.test.ts @@ -35,4 +35,11 @@ describe('parseQuery', () => { regexp: 'x|y', }); }); + + it('returns query for selector with two labels', () => { + expect(parseQuery('{foo="bar", baz="42"}')).toEqual({ + query: '{foo="bar", baz="42"}', + regexp: '', + }); + }); }); diff --git a/public/app/plugins/datasource/logging/datasource.ts b/public/app/plugins/datasource/logging/datasource.ts index 494dcd78d6c..e36da73cd66 100644 --- a/public/app/plugins/datasource/logging/datasource.ts +++ b/public/app/plugins/datasource/logging/datasource.ts @@ -16,26 +16,15 @@ const DEFAULT_QUERY_PARAMS = { query: '', }; -const QUERY_REGEXP = /({\w+="[^"]+"})?\s*(\w[^{]+)?\s*({\w+="[^"]+"})?/; +const selectorRegexp = /{[^{]*}/g; export function parseQuery(input: string) { - const match = input.match(QUERY_REGEXP); + const match = input.match(selectorRegexp); let query = ''; - let regexp = ''; + let regexp = input; if (match) { - if (match[1]) { - query = match[1]; - } - if (match[2]) { - regexp = match[2].trim(); - } - if (match[3]) { - if (match[1]) { - query = `${match[1].slice(0, -1)},${match[3].slice(1)}`; - } else { - query = match[3]; - } - } + query = match[0]; + regexp = input.replace(selectorRegexp, '').trim(); } return { query, regexp }; From c3b67f3a13f1ecb24bfc65c3377cb7a3329120ba Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 28 Nov 2018 10:46:35 +0100 Subject: [PATCH 10/39] Explore: Logging label filtering - adds a custom label renderer to Logs viewer in Explore - labels are no longer treated as strings, they are passed as parsed objects to the log row - label renderer supports onClick handler for an action - renamed Explore's `onClickTableCell` to `onClickLabel` and wired up log label renderers - reuse Prometheus `addLabelToSelector` to modify Logging queries via click on label - added tests to `addLabelToSelector`, changed to include the surrounding `{}` - use label render also for common labels in the controls panel - logging meta data section has now a custom renderer that can render numbers, strings, and labels - style adjustments --- public/app/core/logs_model.ts | 15 +++- public/app/features/explore/Explore.tsx | 7 +- public/app/features/explore/Logs.tsx | 84 +++++++++++++++++-- .../plugins/datasource/logging/datasource.ts | 31 +++---- ...datasource.test.ts => query_utils.test.ts} | 2 +- .../plugins/datasource/logging/query_utils.ts | 17 ++++ .../logging/result_transformer.test.ts | 30 +++---- .../datasource/logging/result_transformer.ts | 25 ++++-- .../prometheus/add_label_to_query.ts | 8 +- .../specs/add_label_to_query.test.ts | 15 +++- public/sass/pages/_explore.scss | 31 ++++++- 11 files changed, 206 insertions(+), 59 deletions(-) rename public/app/plugins/datasource/logging/{datasource.test.ts => query_utils.test.ts} (95%) create mode 100644 public/app/plugins/datasource/logging/query_utils.ts diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 2485dbda40a..a65669c088d 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -35,19 +35,26 @@ export interface LogRow { duplicates?: number; entry: string; key: string; // timestamp + labels - labels: string; + labels: LogsStreamLabels; logLevel: LogLevel; searchWords?: string[]; timestamp: string; // ISO with nanosec precision timeFromNow: string; timeEpochMs: number; timeLocal: string; - uniqueLabels?: string; + uniqueLabels?: LogsStreamLabels; +} + +export enum LogsMetaKind { + Number, + String, + LabelsMap, } export interface LogsMetaItem { label: string; - value: string; + value: string | number | LogsStreamLabels; + kind: LogsMetaKind; } export interface LogsModel { @@ -61,7 +68,7 @@ export interface LogsStream { entries: LogsStreamEntry[]; search?: string; parsedLabels?: LogsStreamLabels; - uniqueLabels?: string; + uniqueLabels?: LogsStreamLabels; } export interface LogsStreamEntry { diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index 44380877c34..317017ae072 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -429,8 +429,8 @@ export class Explore extends React.PureComponent { ); }; - onClickTableCell = (columnKey: string, rowValue: string) => { - this.onModifyQueries({ type: 'ADD_FILTER', key: columnKey, value: rowValue }); + onClickLabel = (key: string, value: string) => { + this.onModifyQueries({ type: 'ADD_FILTER', key, value }); }; onModifyQueries = (action, index?: number) => { @@ -931,7 +931,7 @@ export class Explore extends React.PureComponent { isOpen={showingTable} onToggle={this.onClickTableButton} > - +
)} {supportsLogs && ( @@ -941,6 +941,7 @@ export class Explore extends React.PureComponent { loading={logsLoading} position={position} onChangeTime={this.onChangeTime} + onClickLabel={this.onClickLabel} onStartScanning={this.onStartScanning} onStopScanning={this.onStopScanning} range={range} diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 58965df4514..2c78d9782b9 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -1,9 +1,18 @@ +import _ from 'lodash'; import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; import * as rangeUtil from 'app/core/utils/rangeutil'; import { RawTimeRange } from 'app/types/series'; -import { LogsDedupStrategy, LogsModel, dedupLogRows, filterLogLevels, LogLevel } from 'app/core/logs_model'; +import { + LogsDedupStrategy, + LogsModel, + dedupLogRows, + filterLogLevels, + LogLevel, + LogsStreamLabels, + LogsMetaKind, +} from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; import { Switch } from 'app/core/components/Switch/Switch'; @@ -23,6 +32,51 @@ const graphOptions = { }, }; +function renderMetaItem(value: any, kind: LogsMetaKind) { + if (kind === LogsMetaKind.LabelsMap) { + return ( + + + + ); + } + return value; +} + +class Label extends PureComponent<{ + label: string; + value: string; + onClickLabel?: (label: string, value: string) => void; +}> { + onClickLabel = () => { + const { onClickLabel, label, value } = this.props; + if (onClickLabel) { + onClickLabel(label, value); + } + }; + + render() { + const { label, value } = this.props; + const tooltip = `${label}: ${value}`; + return ( + + {value} + + ); + } +} +class Labels extends PureComponent<{ + labels: LogsStreamLabels; + onClickLabel?: (label: string, value: string) => void; +}> { + render() { + const { labels, onClickLabel } = this.props; + return Object.keys(labels).map(key => ( +