From 3fd810417fc4ba5a2afb016bfd6bcce75754dae1 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 15 Jul 2020 15:20:39 +0200 Subject: [PATCH] Elasticsearch: Create Raw Doc metric to render raw JSON docs in columns in the new table panel (#26233) * test * WIP: Create v2 version * Update tests, remove conosole logs, refactor * Remove incorrect types * Update type * Rename legacy and new metrics * Update * Run request when Raw Data tto Raw Document switch * Fix size updating * Remove _source field from table results as we are showing each source field as column * Remove _source just for metrics, not logs * Revert "Remove _source just for metrics, not logs" This reverts commit 611b6922f762afa0e76fd8679a9b8160bca74e6a. * Revert "Remove _source field from table results as we are showing each source field as column" This reverts commit 31a9d5f81b79b91a12a7e0b74f172ff8afc4bdd3. * Add vis preference for logs * Update visualisation to logs * Revert "Revert "Remove _source just for metrics"" This reverts commit a102ab2894e7a9ca6eee307aa9b60dfea169c718. Co-authored-by: Marcus Efraimsson --- .../elasticsearch/elastic_response.ts | 148 +++++++++++------- .../datasource/elasticsearch/metric_agg.ts | 8 +- .../elasticsearch/partials/metric_agg.html | 2 +- .../datasource/elasticsearch/query_builder.ts | 5 +- .../datasource/elasticsearch/query_ctrl.ts | 6 + .../datasource/elasticsearch/query_def.ts | 3 +- .../elasticsearch/specs/query_def.test.ts | 8 +- 7 files changed, 110 insertions(+), 70 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.ts b/public/app/plugins/datasource/elasticsearch/elastic_response.ts index bb06b2b0537..9d9c1dff0aa 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.ts +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.ts @@ -391,6 +391,88 @@ export class ElasticResponse { } getTimeSeries() { + if (this.targets.some((target: any) => target.metrics.some((metric: any) => metric.type === 'raw_data'))) { + return this.processResponseToDataFrames(false); + } + return this.processResponseToSeries(); + } + + getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse { + return this.processResponseToDataFrames(true, logMessageField, logLevelField); + } + + processResponseToDataFrames( + isLogsRequest: boolean, + logMessageField?: string, + logLevelField?: string + ): DataQueryResponse { + const dataFrame: DataFrame[] = []; + + for (let n = 0; n < this.response.responses.length; n++) { + const response = this.response.responses[n]; + if (response.error) { + throw this.getErrorFromElasticResponse(this.response, response.error); + } + + if (response.hits && response.hits.hits.length > 0) { + const { propNames, docs } = flattenHits(response.hits.hits); + if (docs.length > 0) { + let series = createEmptyDataFrame( + propNames, + this.targets[0].timeField, + isLogsRequest, + logMessageField, + logLevelField + ); + + // Add a row for each document + for (const doc of docs) { + if (logLevelField) { + // Remap level field based on the datasource config. This field is then used in explore to figure out the + // log level. We may rewrite some actual data in the level field if they are different. + doc['level'] = doc[logLevelField]; + } + + series.add(doc); + } + if (isLogsRequest) { + series = addPreferredVisualisationType(series, 'logs'); + } + dataFrame.push(series); + } + } + + if (response.aggregations) { + const aggregations = response.aggregations; + const target = this.targets[n]; + const tmpSeriesList: any[] = []; + const table = new TableModel(); + + this.processBuckets(aggregations, target, tmpSeriesList, table, {}, 0); + this.trimDatapoints(tmpSeriesList, target); + this.nameSeries(tmpSeriesList, target); + + if (table.rows.length > 0) { + dataFrame.push(toDataFrame(table)); + } + + for (let y = 0; y < tmpSeriesList.length; y++) { + let series = toDataFrame(tmpSeriesList[y]); + + // When log results, show aggregations only in graph. Log fields are then going to be shown in table. + if (isLogsRequest) { + series = addPreferredVisualisationType(series, 'graph'); + } + + dataFrame.push(series); + } + } + } + + return { data: dataFrame }; + } + + processResponseToSeries = () => { const seriesList = []; for (let i = 0; i < this.response.responses.length; i++) { @@ -424,59 +506,7 @@ export class ElasticResponse { } return { data: seriesList }; - } - - getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse { - const dataFrame: DataFrame[] = []; - - for (let n = 0; n < this.response.responses.length; n++) { - const response = this.response.responses[n]; - if (response.error) { - throw this.getErrorFromElasticResponse(this.response, response.error); - } - - const { propNames, docs } = flattenHits(response.hits.hits); - if (docs.length > 0) { - let series = createEmptyDataFrame(propNames, this.targets[0].timeField, logMessageField, logLevelField); - - // Add a row for each document - for (const doc of docs) { - if (logLevelField) { - // Remap level field based on the datasource config. This field is then used in explore to figure out the - // log level. We may rewrite some actual data in the level field if they are different. - doc['level'] = doc[logLevelField]; - } - - series.add(doc); - } - - series = addPreferredVisualisationType(series, 'logs'); - dataFrame.push(series); - } - - if (response.aggregations) { - const aggregations = response.aggregations; - const target = this.targets[n]; - const tmpSeriesList: any[] = []; - const table = new TableModel(); - - this.processBuckets(aggregations, target, tmpSeriesList, table, {}, 0); - this.trimDatapoints(tmpSeriesList, target); - this.nameSeries(tmpSeriesList, target); - - for (let y = 0; y < tmpSeriesList.length; y++) { - let series = toDataFrame(tmpSeriesList[y]); - - // When log results, show aggregations only in graph. Log fields are then going to be shown in table. - series = addPreferredVisualisationType(series, 'graph'); - - dataFrame.push(series); - } - } - } - - return { data: dataFrame }; - } + }; } type Doc = { @@ -532,6 +562,7 @@ const flattenHits = (hits: Doc[]): { docs: Array>; propNames const createEmptyDataFrame = ( propNames: string[], timeField: string, + isLogsRequest: boolean, logMessageField?: string, logLevelField?: string ): MutableDataFrame => { @@ -549,13 +580,6 @@ const createEmptyDataFrame = ( }).parse = (v: any) => { return v || ''; }; - } else { - series.addField({ - name: '_source', - type: FieldType.string, - }).parse = (v: any) => { - return JSON.stringify(v, null, 2); - }; } if (logLevelField) { @@ -574,6 +598,10 @@ const createEmptyDataFrame = ( if (fieldNames.includes(propName)) { continue; } + // Do not add _source field (besides logs) as we are showing each _source field in table instead. + if (!isLogsRequest && propName === '_source') { + continue; + } series.addField({ name: propName, diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.ts b/public/app/plugins/datasource/elasticsearch/metric_agg.ts index 0b51b6c0912..7275c4293fb 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.ts +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.ts @@ -101,7 +101,8 @@ export class ElasticMetricAggCtrl { $scope.updateMovingAvgModelSettings(); break; } - case 'raw_document': { + case 'raw_document': + case 'raw_data': { $scope.agg.settings.size = $scope.agg.settings.size || 500; $scope.settingsLinkText = 'Size: ' + $scope.agg.settings.size; $scope.target.metrics.splice(0, $scope.target.metrics.length, $scope.agg); @@ -164,7 +165,10 @@ export class ElasticMetricAggCtrl { $scope.showOptions = false; // reset back to metric/group by query - if ($scope.target.bucketAggs.length === 0 && $scope.agg.type !== 'raw_document') { + if ( + $scope.target.bucketAggs.length === 0 && + ($scope.agg.type !== 'raw_document' || $scope.agg.type !== 'raw_data') + ) { $scope.target.bucketAggs = [queryDef.defaultBucketAgg()]; } diff --git a/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html b/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html index 8f3eedd6a55..5e515bc57a8 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html @@ -101,7 +101,7 @@ -
+
diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index fa61085685b..98c994cbfcd 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -212,7 +212,8 @@ export class ElasticQueryBuilder { // If target doesn't have bucketAggs and type is not raw_document, it is invalid query. if (target.bucketAggs.length === 0) { metric = target.metrics[0]; - if (!metric || metric.type !== 'raw_document') { + + if (!metric || !(metric.type === 'raw_document' || metric.type === 'raw_data')) { throw { message: 'Invalid query' }; } } @@ -221,7 +222,7 @@ export class ElasticQueryBuilder { * Check if metric type is raw_document. If metric doesn't have size (or size is 0), update size to 500. * Otherwise it will not be a valid query and error will be thrown. */ - if (target.metrics?.[0]?.type === 'raw_document') { + if (target.metrics?.[0]?.type === 'raw_document' || target.metrics?.[0]?.type === 'raw_data') { metric = target.metrics[0]; const size = (metric.settings && metric.settings.size !== 0 && metric.settings.size) || 500; return this.documentQuery(query, size); diff --git a/public/app/plugins/datasource/elasticsearch/query_ctrl.ts b/public/app/plugins/datasource/elasticsearch/query_ctrl.ts index 85e8158734f..fa5831c8c4e 100644 --- a/public/app/plugins/datasource/elasticsearch/query_ctrl.ts +++ b/public/app/plugins/datasource/elasticsearch/query_ctrl.ts @@ -51,9 +51,15 @@ export class ElasticQueryCtrl extends QueryCtrl { } queryUpdated() { + // As Raw Data and Raw Document have the same request, we need to run refresh if they are updated + const isPossiblyRawDataSwitch = this.target.metrics.some( + (metric: any) => metric.type === 'raw_data' || metric.type === 'raw_document' + ); const newJson = angular.toJson(this.datasource.queryBuilder.build(this.target), true); if (this.rawQueryOld && newJson !== this.rawQueryOld) { this.refresh(); + } else if (isPossiblyRawDataSwitch) { + this.refresh(); } this.rawQueryOld = newJson; diff --git a/public/app/plugins/datasource/elasticsearch/query_def.ts b/public/app/plugins/datasource/elasticsearch/query_def.ts index e85feadd360..23b015839a8 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.ts +++ b/public/app/plugins/datasource/elasticsearch/query_def.ts @@ -79,7 +79,8 @@ export const metricAggTypes = [ supportsMultipleBucketPaths: true, minVersion: 2, }, - { text: 'Raw Document', value: 'raw_document', requiresField: false }, + { text: 'Raw Document (legacy)', value: 'raw_document', requiresField: false }, + { text: 'Raw Data', value: 'raw_data', requiresField: false }, { text: 'Logs', value: 'logs', requiresField: false }, ]; diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts index 1b94965ceb4..6ce182e1903 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts @@ -92,25 +92,25 @@ describe('ElasticQueryDef', () => { describe('pipeline aggs depending on esverison', () => { describe('using esversion undefined', () => { test('should not get pipeline aggs', () => { - expect(queryDef.getMetricAggTypes(undefined).length).toBe(10); + expect(queryDef.getMetricAggTypes(undefined).length).toBe(11); }); }); describe('using esversion 1', () => { test('should not get pipeline aggs', () => { - expect(queryDef.getMetricAggTypes(1).length).toBe(10); + expect(queryDef.getMetricAggTypes(1).length).toBe(11); }); }); describe('using esversion 2', () => { test('should get pipeline aggs', () => { - expect(queryDef.getMetricAggTypes(2).length).toBe(14); + expect(queryDef.getMetricAggTypes(2).length).toBe(15); }); }); describe('using esversion 5', () => { test('should get pipeline aggs', () => { - expect(queryDef.getMetricAggTypes(5).length).toBe(14); + expect(queryDef.getMetricAggTypes(5).length).toBe(15); }); }); });