From 982548417681d8cf92b40eb052ede7e8b1cc7bd4 Mon Sep 17 00:00:00 2001 From: Mattia Rossi Date: Sat, 24 Nov 2018 10:48:29 -0500 Subject: [PATCH 01/31] Add support for Offset in elasticsearch datasource, date_histogram aggregation, fixes grafana #12653 --- .../datasource/elasticsearch/partials/bucket_agg.html | 10 ++++++++++ .../plugins/datasource/elasticsearch/query_builder.ts | 1 + 2 files changed, 11 insertions(+) diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index d0cf23a3d93..a6f68685681 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -70,6 +70,16 @@ +
+ + +
+
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index efdeee68370..7d9490521e3 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -71,6 +71,7 @@ export class ElasticQueryBuilder { esAgg.min_doc_count = settings.min_doc_count || 0; esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' }; esAgg.format = 'epoch_millis'; + esAgg.offset = settings.offset || '0'; if (esAgg.interval === 'auto') { esAgg.interval = '$__interval'; From f6fc7f7b64f7f51e519c722459f849765f6d55fb Mon Sep 17 00:00:00 2001 From: Mattia Rossi Date: Mon, 26 Nov 2018 08:58:25 -0500 Subject: [PATCH 02/31] Requested Backend changes, added details to popover description for the offset field --- pkg/tsdb/elasticsearch/client/models.go | 1 + pkg/tsdb/elasticsearch/time_series_query.go | 3 +++ .../datasource/elasticsearch/partials/bucket_agg.html | 3 ++- public/app/plugins/datasource/elasticsearch/query_builder.ts | 5 ++++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index a0d257d01a6..5307dc34b27 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -240,6 +240,7 @@ type DateHistogramAgg struct { Missing *string `json:"missing,omitempty"` ExtendedBounds *ExtendedBounds `json:"extended_bounds"` Format string `json:"format"` + Offset string `json:"offset,omitempty"` } // FiltersAggregation represents a filters aggregation diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 869e23e21ce..3e43a26089a 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -134,6 +134,9 @@ func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFro a.Interval = "$__interval" } + if bucketAgg.Settings.Get("offset").MustString("") != "" { + a.Offset = bucketAgg.Settings.Get("offset").MustString("") + } if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { a.Missing = &missing } diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index a6f68685681..9d6b8196e8e 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -74,7 +74,8 @@ diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index 7d9490521e3..c505d402874 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -71,7 +71,10 @@ export class ElasticQueryBuilder { esAgg.min_doc_count = settings.min_doc_count || 0; esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' }; esAgg.format = 'epoch_millis'; - esAgg.offset = settings.offset || '0'; + + if (settings.offset !== '') { + esAgg.offset = settings.offset; + } if (esAgg.interval === 'auto') { esAgg.interval = '$__interval'; From ce74b1ddc23fd3ab94dbd58064c16f85f49119a9 Mon Sep 17 00:00:00 2001 From: Mattia Rossi Date: Tue, 27 Nov 2018 03:42:20 -0500 Subject: [PATCH 03/31] Requested Backend changes, removed link in popover description for the offset field --- pkg/tsdb/elasticsearch/time_series_query.go | 5 +++-- .../datasource/elasticsearch/partials/bucket_agg.html | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 3e43a26089a..a25c3cf693d 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -134,9 +134,10 @@ func addDateHistogramAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, timeFro a.Interval = "$__interval" } - if bucketAgg.Settings.Get("offset").MustString("") != "" { - a.Offset = bucketAgg.Settings.Get("offset").MustString("") + if offset, err := bucketAgg.Settings.Get("offset").String(); err == nil { + a.Offset = offset } + if missing, err := bucketAgg.Settings.Get("missing").String(); err == nil { a.Missing = &missing } diff --git a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html index 9d6b8196e8e..b17f03bced3 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html @@ -74,8 +74,7 @@ From c3b67f3a13f1ecb24bfc65c3377cb7a3329120ba Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Wed, 28 Nov 2018 10:46:35 +0100 Subject: [PATCH 04/31] Explore: Logging label filtering - adds a custom label renderer to Logs viewer in Explore - labels are no longer treated as strings, they are passed as parsed objects to the log row - label renderer supports onClick handler for an action - renamed Explore's `onClickTableCell` to `onClickLabel` and wired up log label renderers - reuse Prometheus `addLabelToSelector` to modify Logging queries via click on label - added tests to `addLabelToSelector`, changed to include the surrounding `{}` - use label render also for common labels in the controls panel - logging meta data section has now a custom renderer that can render numbers, strings, and labels - style adjustments --- public/app/core/logs_model.ts | 15 +++- public/app/features/explore/Explore.tsx | 7 +- public/app/features/explore/Logs.tsx | 84 +++++++++++++++++-- .../plugins/datasource/logging/datasource.ts | 31 +++---- ...datasource.test.ts => query_utils.test.ts} | 2 +- .../plugins/datasource/logging/query_utils.ts | 17 ++++ .../logging/result_transformer.test.ts | 30 +++---- .../datasource/logging/result_transformer.ts | 25 ++++-- .../prometheus/add_label_to_query.ts | 8 +- .../specs/add_label_to_query.test.ts | 15 +++- public/sass/pages/_explore.scss | 31 ++++++- 11 files changed, 206 insertions(+), 59 deletions(-) rename public/app/plugins/datasource/logging/{datasource.test.ts => query_utils.test.ts} (95%) create mode 100644 public/app/plugins/datasource/logging/query_utils.ts diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 2485dbda40a..a65669c088d 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -35,19 +35,26 @@ export interface LogRow { duplicates?: number; entry: string; key: string; // timestamp + labels - labels: string; + labels: LogsStreamLabels; logLevel: LogLevel; searchWords?: string[]; timestamp: string; // ISO with nanosec precision timeFromNow: string; timeEpochMs: number; timeLocal: string; - uniqueLabels?: string; + uniqueLabels?: LogsStreamLabels; +} + +export enum LogsMetaKind { + Number, + String, + LabelsMap, } export interface LogsMetaItem { label: string; - value: string; + value: string | number | LogsStreamLabels; + kind: LogsMetaKind; } export interface LogsModel { @@ -61,7 +68,7 @@ export interface LogsStream { entries: LogsStreamEntry[]; search?: string; parsedLabels?: LogsStreamLabels; - uniqueLabels?: string; + uniqueLabels?: LogsStreamLabels; } export interface LogsStreamEntry { diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index 44380877c34..317017ae072 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -429,8 +429,8 @@ export class Explore extends React.PureComponent { ); }; - onClickTableCell = (columnKey: string, rowValue: string) => { - this.onModifyQueries({ type: 'ADD_FILTER', key: columnKey, value: rowValue }); + onClickLabel = (key: string, value: string) => { + this.onModifyQueries({ type: 'ADD_FILTER', key, value }); }; onModifyQueries = (action, index?: number) => { @@ -931,7 +931,7 @@ export class Explore extends React.PureComponent { isOpen={showingTable} onToggle={this.onClickTableButton} > - +
)} {supportsLogs && ( @@ -941,6 +941,7 @@ export class Explore extends React.PureComponent { loading={logsLoading} position={position} onChangeTime={this.onChangeTime} + onClickLabel={this.onClickLabel} onStartScanning={this.onStartScanning} onStopScanning={this.onStopScanning} range={range} diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 58965df4514..2c78d9782b9 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -1,9 +1,18 @@ +import _ from 'lodash'; import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; import * as rangeUtil from 'app/core/utils/rangeutil'; import { RawTimeRange } from 'app/types/series'; -import { LogsDedupStrategy, LogsModel, dedupLogRows, filterLogLevels, LogLevel } from 'app/core/logs_model'; +import { + LogsDedupStrategy, + LogsModel, + dedupLogRows, + filterLogLevels, + LogLevel, + LogsStreamLabels, + LogsMetaKind, +} from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; import { Switch } from 'app/core/components/Switch/Switch'; @@ -23,6 +32,51 @@ const graphOptions = { }, }; +function renderMetaItem(value: any, kind: LogsMetaKind) { + if (kind === LogsMetaKind.LabelsMap) { + return ( + + + + ); + } + return value; +} + +class Label extends PureComponent<{ + label: string; + value: string; + onClickLabel?: (label: string, value: string) => void; +}> { + onClickLabel = () => { + const { onClickLabel, label, value } = this.props; + if (onClickLabel) { + onClickLabel(label, value); + } + }; + + render() { + const { label, value } = this.props; + const tooltip = `${label}: ${value}`; + return ( + + {value} + + ); + } +} +class Labels extends PureComponent<{ + labels: LogsStreamLabels; + onClickLabel?: (label: string, value: string) => void; +}> { + render() { + const { labels, onClickLabel } = this.props; + return Object.keys(labels).map(key => ( +