From 491a4dc9674058df8c299b02a7f0738e76eea525 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 10 Dec 2020 04:19:14 -0700 Subject: [PATCH] Elasticsearch: View in context feature for logs (#28764) * Elasticsearch: View in context feature for logs * Fixing unused type * Limit show context to esVersion > 5 * Fixing scope for showContextToggle; removing console.log * Fix typing; adding check for lineField * Update test to reflect new sorting keys for logs * Removing sort from metadata fields * Adding comment for clarity * Fixing scenerio where the data is missing * remove export & use optional chaining Co-authored-by: Elfo404 --- public/app/features/explore/LogsContainer.tsx | 12 ++- .../datasource/elasticsearch/datasource.ts | 90 +++++++++++++++++++ .../elasticsearch/elastic_response.ts | 3 + .../datasource/elasticsearch/query_builder.ts | 10 ++- .../elasticsearch/specs/query_builder.test.ts | 12 ++- 5 files changed, 117 insertions(+), 10 deletions(-) diff --git a/public/app/features/explore/LogsContainer.tsx b/public/app/features/explore/LogsContainer.tsx index 56397a1955d..1b2fc53a27c 100644 --- a/public/app/features/explore/LogsContainer.tsx +++ b/public/app/features/explore/LogsContainer.tsx @@ -91,6 +91,16 @@ export class LogsContainer extends PureComponent { return []; }; + showContextToggle = (): boolean => { + const { datasourceInstance } = this.props; + + if (datasourceInstance?.showContextToggle) { + return datasourceInstance.showContextToggle(); + } + + return false; + }; + getFieldLinks = (field: Field, rowIndex: number) => { return getFieldLinksForExplore(field, rowIndex, this.props.splitOpen, this.props.range); }; @@ -157,7 +167,7 @@ export class LogsContainer extends PureComponent { timeZone={timeZone} scanning={scanning} scanRange={range.raw} - showContextToggle={this.props.datasourceInstance?.showContextToggle} + showContextToggle={this.showContextToggle} width={width} getRowContext={this.getLogRowContext} getFieldLinks={this.getFieldLinks} diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index e1835630b63..656511c6e22 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -9,6 +9,8 @@ import { DataLink, PluginMeta, DataQuery, + LogRowModel, + Field, MetricFindValue, } from '@grafana/data'; import LanguageProvider from './language_provider'; @@ -21,6 +23,7 @@ import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types'; +import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider'; import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils'; import { isMetricAggregationWithField, @@ -432,6 +435,74 @@ export class ElasticDatasource extends DataSourceApi 5; + } + + getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => { + const sortField = row.dataFrame.fields.find(f => f.name === 'sort'); + const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs]; + const range = this.timeSrv.timeRange(); + const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc'; + const header = this.getQueryHeader('query_then_fetch', range.from, range.to); + const limit = options?.limit ?? 10; + const esQuery = JSON.stringify({ + size: limit, + query: { + bool: { + filter: [ + { + range: { + [this.timeField]: { + gte: range.from.valueOf(), + lte: range.to.valueOf(), + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + sort: [{ [this.timeField]: direction }, { _doc: direction }], + search_after: searchAfter, + }); + const payload = [header, esQuery].join('\n') + '\n'; + const url = this.getMultiSearchUrl(); + const response = await this.post(url, payload); + const targets: ElasticsearchQuery[] = [{ refId: `${row.dataFrame.refId}`, metrics: [], isLogsQuery: true }]; + const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, direction)); + const logResponse = elasticResponse.getLogs(this.logMessageField, this.logLevelField); + const dataFrame = _.first(logResponse.data); + if (!dataFrame) { + return { data: [] }; + } + /** + * The LogRowContextProvider requires there is a field in the dataFrame.fields + * named `ts` for timestamp and `line` for the actual log line to display. + * Unfortunatly these fields are hardcoded and are required for the lines to + * be properly displayed. This code just copies the fields based on this.timeField + * and this.logMessageField and recreates the dataFrame so it works. + */ + const timestampField = dataFrame.fields.find((f: Field) => f.name === this.timeField); + const lineField = dataFrame.fields.find((f: Field) => f.name === this.logMessageField); + if (timestampField && lineField) { + return { + data: [ + { + ...dataFrame, + fields: [...dataFrame.fields, { ...timestampField, name: 'ts' }, { ...lineField, name: 'line' }], + }, + ], + }; + } + return logResponse; + }; + query(options: DataQueryRequest): Promise { let payload = ''; const targets = this.interpolateVariablesInQueries(_.cloneDeep(options.targets), options.scopedVars); @@ -758,3 +829,22 @@ export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig field.config.links = [...(field.config.links || []), link]; } } + +function transformHitsBasedOnDirection(response: any, direction: 'asc' | 'desc') { + if (direction === 'desc') { + return response; + } + const actualResponse = response.responses[0]; + return { + ...response, + responses: [ + { + ...actualResponse, + hits: { + ...actualResponse.hits, + hits: actualResponse.hits.hits.reverse(), + }, + }, + ], + }; +} diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.ts b/public/app/plugins/datasource/elasticsearch/elastic_response.ts index c7d5f1f9422..b1e282bf823 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.ts +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.ts @@ -372,6 +372,7 @@ export class ElasticResponse { _id: hit._id, _type: hit._type, _index: hit._index, + sort: hit.sort, }; if (hit._source) { @@ -552,6 +553,7 @@ type Doc = { _type: string; _index: string; _source?: any; + sort?: Array; }; /** @@ -572,6 +574,7 @@ const flattenHits = (hits: Doc[]): { docs: Array>; propNames _id: hit._id, _type: hit._type, _index: hit._index, + sort: hit.sort, _source: { ...flattened }, ...flattened, }; diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index 60f2178e5b6..f0180ad7fd8 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -131,8 +131,14 @@ export class ElasticQueryBuilder { documentQuery(query: any, size: number) { query.size = size; - query.sort = {}; - query.sort[this.timeField] = { order: 'desc', unmapped_type: 'boolean' }; + query.sort = [ + { + [this.timeField]: { order: 'desc', unmapped_type: 'boolean' }, + }, + { + _doc: { order: 'desc' }, + }, + ]; // fields field not supported on ES 5.x if (this.esVersion < 5) { diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts index df1f20fedb8..c9347bd1778 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts @@ -243,12 +243,7 @@ describe('ElasticQueryBuilder', () => { ], }, }, - sort: { - '@timestamp': { - order: 'desc', - unmapped_type: 'boolean', - }, - }, + sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }, { _doc: { order: 'desc' } }], script_fields: {}, }); }); @@ -573,7 +568,10 @@ describe('ElasticQueryBuilder', () => { }; expect(query.query).toEqual(expectedQuery); - expect(query.sort).toEqual({ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }); + expect(query.sort).toEqual([ + { '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }, + { _doc: { order: 'desc' } }, + ]); const expectedAggs = { // FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and