From a7238ba933e6cb60dc56f8d5c5419133939c27fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Jamr=C3=B3z?= Date: Tue, 7 Mar 2023 15:00:11 +0100 Subject: [PATCH] Explore: Support mixed data sources for supplementary query (#63036) * Consolidate logs volume logic (full range and limited) * Fix showing limited histogram message * Test passing meta data to logs volume provider * Improve readability * Clean up types * Add basic support for multiple log volumes * Move the comment back to the right place * Improve readability * Clean up the logic to support Logs Samples * Update docs * Sort log volumes * Provide title to logs volume panel * Move logs volume cache to the provider factory * Add helper functions * Reuse only if queries are the same * Fix alphabetical sorting * Move caching out of the provider * Support errors and loading state * Remove unused code * Consolidate supplementary query utils * Add tests for supplementaryQueries * Update tests * Simplify logs volume extra info * Update tests * Remove comment * Update tests * Fix hiding the histogram for hidden queries * Simplify loading message * Update tests * Wait for full fallback histogram to load before showing it * Fix a typo * Add feedback comments * Move feedback comments to github * Do not filter out hidden queries as they may be used as references in other queries * Group log volume by refId * Support showing fallback histograms per query to avoid duplicates * Improve type-checking * Fix supplementaryQueries.test.ts * Fix logsModel.test.ts * Fix loading fallback results * Fix unit tests * WIP * Update deprecated styles * Simplify test * Simplify rendering zoom info * Update deprecated styles * Simplify getLogsVolumeDataSourceInfo * Simplify isLogsVolumeLimited() * Simplify rendering zoom info --- .betterer.results | 4 - packages/grafana-data/src/types/logs.ts | 35 ++ public/app/core/logsModel.test.ts | 74 ++-- public/app/core/logsModel.ts | 124 ++---- public/app/features/explore/Logs.tsx | 16 +- .../app/features/explore/LogsSamplePanel.tsx | 6 +- .../features/explore/LogsVolumePanel.test.tsx | 26 -- .../app/features/explore/LogsVolumePanel.tsx | 82 ++-- .../explore/LogsVolumePanelList.test.tsx | 54 +++ .../features/explore/LogsVolumePanelList.tsx | 120 ++++++ public/app/features/explore/__mocks__/data.ts | 37 ++ public/app/features/explore/state/query.ts | 47 ++- .../explore/state/supplementaryQueries.ts | 49 --- .../features/explore/utils/decorators.test.ts | 119 +----- .../utils/supplementaryQueries.test.ts | 357 ++++++++++++++++++ .../explore/utils/supplementaryQueries.ts | 174 ++++++++- public/test/mocks/datasource_srv.ts | 6 + public/test/mocks/query.ts | 30 ++ 18 files changed, 943 insertions(+), 417 deletions(-) create mode 100644 public/app/features/explore/LogsVolumePanelList.test.tsx create mode 100644 public/app/features/explore/LogsVolumePanelList.tsx create mode 100644 public/app/features/explore/__mocks__/data.ts delete mode 100644 public/app/features/explore/state/supplementaryQueries.ts create mode 100644 public/app/features/explore/utils/supplementaryQueries.test.ts create mode 100644 public/test/mocks/query.ts diff --git a/.betterer.results b/.betterer.results index e3ff6cb8ab5..e04d60a60a3 100644 --- a/.betterer.results +++ b/.betterer.results @@ -2279,10 +2279,6 @@ exports[`better eslint`] = { "public/app/core/history/richHistoryLocalStorageUtils.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], - "public/app/core/logsModel.ts:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"], - [0, 0, 0, "Do not use any type assertions.", "1"] - ], "public/app/core/navigation/GrafanaRoute.test.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"] diff --git a/packages/grafana-data/src/types/logs.ts b/packages/grafana-data/src/types/logs.ts index 14670f63a3d..bc974b961c6 100644 --- a/packages/grafana-data/src/types/logs.ts +++ b/packages/grafana-data/src/types/logs.ts @@ -87,6 +87,7 @@ export interface LogsModel { // visibleRange is time range for histogram created from log results visibleRange?: AbsoluteTimeRange; queries?: DataQuery[]; + bucketSize?: number; } export interface LogSearchMatch { @@ -194,6 +195,40 @@ export enum LogsVolumeType { Limited = 'Limited', } +/** + * Custom meta information required by Logs Volume responses + */ +export type LogsVolumeCustomMetaData = { + absoluteRange: AbsoluteTimeRange; + logsVolumeType: LogsVolumeType; + datasourceName: string; + sourceQuery: DataQuery; +}; + +export const getLogsVolumeAbsoluteRange = ( + dataFrames: DataFrame[], + defaultRange: AbsoluteTimeRange +): AbsoluteTimeRange => { + return dataFrames[0].meta?.custom?.absoluteRange || defaultRange; +}; + +export const getLogsVolumeDataSourceInfo = (dataFrames: DataFrame[]): { name: string; refId: string } | null => { + const customMeta = dataFrames[0]?.meta?.custom; + + if (customMeta && customMeta.datasourceName && customMeta.sourceQuery?.refId) { + return { + name: customMeta.datasourceName, + refId: customMeta.sourceQuery.refId, + }; + } + + return null; +}; + +export const isLogsVolumeLimited = (dataFrames: DataFrame[]) => { + return dataFrames[0]?.meta?.custom?.logsVolumeType === LogsVolumeType.Limited; +}; + /** * Data sources that support supplementary queries in Explore. * This will enable users to see additional data when running original queries. diff --git a/public/app/core/logsModel.test.ts b/public/app/core/logsModel.test.ts index 5df6f2e1dbb..14daad0b413 100644 --- a/public/app/core/logsModel.test.ts +++ b/public/app/core/logsModel.test.ts @@ -29,8 +29,8 @@ import { getSeriesProperties, LIMIT_LABEL, logSeriesToLogsModel, - queryLogsVolume, queryLogsSample, + queryLogsVolume, } from './logsModel'; const FROM = dateTimeParse('2021-06-17 00:00:00', { timeZone: 'utc' }); @@ -1123,8 +1123,9 @@ describe('logs volume', () => { datasource: MockObservableDataSourceApi, request: DataQueryRequest; - function createFrame(labels: object, timestamps: number[], values: number[]) { + function createFrame(labels: object, timestamps: number[], values: number[], refId: string) { return toDataFrame({ + refId, fields: [ { name: 'Time', type: FieldType.time, values: timestamps }, { @@ -1137,20 +1138,13 @@ describe('logs volume', () => { }); } - function createExpectedFields(levelName: string) { - return [ - expect.objectContaining({ name: 'Time' }), - expect.objectContaining({ - name: 'Value', - config: expect.objectContaining({ displayNameFromDS: levelName }), - }), - ]; - } - function setup(datasourceSetup: () => void) { datasourceSetup(); request = { - targets: [{ target: 'volume query 1' }, { target: 'volume query 2' }], + targets: [ + { refId: 'A', target: 'volume query 1' }, + { refId: 'B', target: 'volume query 2' }, + ], scopedVars: {}, } as unknown as DataQueryRequest; volumeProvider = queryLogsVolume(datasource, request, { @@ -1168,19 +1162,21 @@ describe('logs volume', () => { function setupMultipleResults() { // level=unknown - const resultAFrame1 = createFrame({ app: 'app01' }, [100, 200, 300], [5, 5, 5]); + const resultAFrame1 = createFrame({ app: 'app01' }, [100, 200, 300], [5, 5, 5], 'A'); // level=error - const resultAFrame2 = createFrame({ app: 'app01', level: 'error' }, [100, 200, 300], [0, 1, 0]); + const resultAFrame2 = createFrame({ app: 'app01', level: 'error' }, [100, 200, 300], [0, 1, 0], 'B'); // level=unknown - const resultBFrame1 = createFrame({ app: 'app02' }, [100, 200, 300], [1, 2, 3]); + const resultBFrame1 = createFrame({ app: 'app02' }, [100, 200, 300], [1, 2, 3], 'A'); // level=error - const resultBFrame2 = createFrame({ app: 'app02', level: 'error' }, [100, 200, 300], [1, 1, 1]); + const resultBFrame2 = createFrame({ app: 'app02', level: 'error' }, [100, 200, 300], [1, 1, 1], 'B'); datasource = new MockObservableDataSourceApi('loki', [ { + state: LoadingState.Loading, data: [resultAFrame1, resultAFrame2], }, { + state: LoadingState.Done, data: [resultBFrame1, resultBFrame2], }, ]); @@ -1188,9 +1184,9 @@ describe('logs volume', () => { function setupMultipleResultsStreaming() { // level=unknown - const resultAFrame1 = createFrame({ app: 'app01' }, [100, 200, 300], [5, 5, 5]); + const resultAFrame1 = createFrame({ app: 'app01' }, [100, 200, 300], [5, 5, 5], 'A'); // level=error - const resultAFrame2 = createFrame({ app: 'app01', level: 'error' }, [100, 200, 300], [0, 1, 0]); + const resultAFrame2 = createFrame({ app: 'app01', level: 'error' }, [100, 200, 300], [0, 1, 0], 'B'); datasource = new MockObservableDataSourceApi('loki', [ { @@ -1198,7 +1194,7 @@ describe('logs volume', () => { data: [resultAFrame1], }, { - state: LoadingState.Streaming, + state: LoadingState.Done, data: [resultAFrame1, resultAFrame2], }, ]); @@ -1221,14 +1217,8 @@ describe('logs volume', () => { fields: expect.anything(), meta: { custom: { - targets: [ - { - target: 'volume query 1', - }, - { - target: 'volume query 2', - }, - ], + sourceQuery: { refId: 'A', target: 'volume query 1' }, + datasourceName: 'loki', logsVolumeType: LogsVolumeType.FullRange, absoluteRange: { from: FROM.valueOf(), @@ -1243,7 +1233,7 @@ describe('logs volume', () => { }); }); - it('applies correct meta datya when streaming', async () => { + it('applies correct meta data when streaming', async () => { setup(setupMultipleResultsStreaming); await expect(volumeProvider).toEmitValuesWith((received) => { @@ -1256,14 +1246,8 @@ describe('logs volume', () => { fields: expect.anything(), meta: { custom: { - targets: [ - { - target: 'volume query 1', - }, - { - target: 'volume query 2', - }, - ], + sourceQuery: { refId: 'A', target: 'volume query 1' }, + datasourceName: 'loki', logsVolumeType: LogsVolumeType.FullRange, absoluteRange: { from: FROM.valueOf(), @@ -1278,22 +1262,6 @@ describe('logs volume', () => { }); }); - it('aggregates data frames by level', async () => { - setup(setupMultipleResults); - - await expect(volumeProvider).toEmitValuesWith((received) => { - expect(received).toContainEqual({ - state: LoadingState.Done, - error: undefined, - data: expect.arrayContaining([ - expect.objectContaining({ - fields: expect.arrayContaining(createExpectedFields('error')), - }), - ]), - }); - }); - }); - it('returns error', async () => { setup(setupErrorResponse); diff --git a/public/app/core/logsModel.ts b/public/app/core/logsModel.ts index 04177206e3f..85453bab181 100644 --- a/public/app/core/logsModel.ts +++ b/public/app/core/logsModel.ts @@ -1,4 +1,4 @@ -import { size } from 'lodash'; +import { groupBy, size } from 'lodash'; import { from, isObservable, Observable } from 'rxjs'; import { @@ -13,7 +13,6 @@ import { dateTimeFormatTimeAgo, FieldCache, FieldColorModeId, - FieldConfig, FieldType, FieldWithIndex, findCommonLabels, @@ -27,8 +26,8 @@ import { LogsMetaItem, LogsMetaKind, LogsModel, + LogsVolumeCustomMetaData, LogsVolumeType, - MutableDataFrame, rangeUtil, ScopedVars, sortDataFrame, @@ -225,6 +224,7 @@ export function dataFrameToLogsModel( absoluteRange ); logsModel.visibleRange = visibleRange; + logsModel.bucketSize = bucketSize; logsModel.series = makeDataFramesForLogs(sortedRows, bucketSize); if (logsModel.meta) { @@ -605,69 +605,22 @@ function getLogVolumeFieldConfig(level: LogLevel, oneLevelDetected: boolean) { }; } -/** - * Take multiple data frames, sum up values and group by level. - * Return a list of data frames, each representing single level. - */ -export function aggregateRawLogsVolume( - rawLogsVolume: DataFrame[], - extractLevel: (dataFrame: DataFrame) => LogLevel -): DataFrame[] { - const logsVolumeByLevelMap: Partial> = {}; - - rawLogsVolume.forEach((dataFrame) => { - const level = extractLevel(dataFrame); - if (!logsVolumeByLevelMap[level]) { - logsVolumeByLevelMap[level] = []; +const updateLogsVolumeConfig = ( + dataFrame: DataFrame, + extractLevel: (dataFrame: DataFrame) => LogLevel, + oneLevelDetected: boolean +): DataFrame => { + dataFrame.fields = dataFrame.fields.map((field) => { + if (field.type === FieldType.number) { + field.config = { + ...field.config, + ...getLogVolumeFieldConfig(extractLevel(dataFrame), oneLevelDetected), + }; } - logsVolumeByLevelMap[level]!.push(dataFrame); + return field; }); - - return Object.keys(logsVolumeByLevelMap).map((level: string) => { - return aggregateFields( - logsVolumeByLevelMap[level as LogLevel]!, - getLogVolumeFieldConfig(level as LogLevel, Object.keys(logsVolumeByLevelMap).length === 1) - ); - }); -} - -/** - * Aggregate multiple data frames into a single data frame by adding values. - * Multiple data frames for the same level are passed here to get a single - * data frame for a given level. Aggregation by level happens in aggregateRawLogsVolume() - */ -function aggregateFields(dataFrames: DataFrame[], config: FieldConfig): DataFrame { - const aggregatedDataFrame = new MutableDataFrame(); - if (!dataFrames.length) { - return aggregatedDataFrame; - } - - const totalLength = dataFrames[0].length; - const timeField = new FieldCache(dataFrames[0]).getFirstFieldOfType(FieldType.time); - - if (!timeField) { - return aggregatedDataFrame; - } - - aggregatedDataFrame.addField({ name: 'Time', type: FieldType.time }, totalLength); - aggregatedDataFrame.addField({ name: 'Value', type: FieldType.number, config }, totalLength); - - dataFrames.forEach((dataFrame) => { - dataFrame.fields.forEach((field) => { - if (field.type === FieldType.number) { - for (let pointIndex = 0; pointIndex < totalLength; pointIndex++) { - const currentValue = aggregatedDataFrame.get(pointIndex).Value; - const valueToAdd = field.values.get(pointIndex); - const totalValue = - currentValue === null && valueToAdd === null ? null : (currentValue || 0) + (valueToAdd || 0); - aggregatedDataFrame.set(pointIndex, { Value: totalValue, Time: timeField.values.get(pointIndex) }); - } - } - }); - }); - - return aggregatedDataFrame; -} + return dataFrame; +}; type LogsVolumeQueryOptions = { extractLevel: (dataFrame: DataFrame) => LogLevel; @@ -697,7 +650,7 @@ export function queryLogsVolume { - let rawLogsVolume: DataFrame[] = []; + let logsVolumeData: DataFrame[] = []; observer.next({ state: LoadingState.Loading, error: undefined, @@ -709,11 +662,6 @@ export function queryLogsVolume { - observer.next({ - state: LoadingState.Done, - error: undefined, - data: rawLogsVolume, - }); observer.complete(); }, next: (dataQueryResponse: DataQueryResponse) => { @@ -726,24 +674,34 @@ export function queryLogsVolume { + let sourceRefId = dataFrame.refId || ''; + if (sourceRefId.startsWith('log-volume-')) { + sourceRefId = sourceRefId.substr('log-volume-'.length); + } + + const logsVolumeCustomMetaData: LogsVolumeCustomMetaData = { + logsVolumeType: LogsVolumeType.FullRange, + absoluteRange: { from: options.range.from.valueOf(), to: options.range.to.valueOf() }, + datasourceName: datasource.name, + sourceQuery: options.targets.find((dataQuery) => dataQuery.refId === sourceRefId)!, + }; + + dataFrame.meta = { + ...dataFrame.meta, custom: { - targets: options.targets, - logsVolumeType: LogsVolumeType.FullRange, - absoluteRange: { from: options.range.from.valueOf(), to: options.range.to.valueOf() }, + ...dataFrame.meta?.custom, + ...logsVolumeCustomMetaData, }, }; - } - rawLogsVolume = aggregatedLogsVolume; + return updateLogsVolumeConfig(dataFrame, options.extractLevel, framesByRefId[dataFrame.refId].length === 1); + }); + observer.next({ - state: dataQueryResponse.state ?? LoadingState.Streaming, + state: dataQueryResponse.state, error: undefined, - data: rawLogsVolume, + data: logsVolumeData, }); } }, diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index f934b8b9dda..d92a44cca3a 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -48,7 +48,7 @@ import { LogRows } from '../logs/components/LogRows'; import { LogsMetaRow } from './LogsMetaRow'; import LogsNavigation from './LogsNavigation'; -import { LogsVolumePanel } from './LogsVolumePanel'; +import { LogsVolumePanelList } from './LogsVolumePanelList'; import { SETTINGS_KEYS } from './utils/logs'; interface Props extends Themeable2 { @@ -324,13 +324,10 @@ class UnthemedLogs extends PureComponent { splitOpen, logRows, logsMeta, - logsSeries, - visibleRange, logsVolumeEnabled, logsVolumeData, loadLogsVolumeData, loading = false, - loadingState, onClickFilterLabel, onClickFilterOutLabel, timeZone, @@ -377,19 +374,10 @@ class UnthemedLogs extends PureComponent { <> {logsVolumeEnabled && ( - {LogsSamplePanelContent} - ); + ) : null; } const getStyles = (theme: GrafanaTheme2) => ({ diff --git a/public/app/features/explore/LogsVolumePanel.test.tsx b/public/app/features/explore/LogsVolumePanel.test.tsx index c47b1caae15..54ebf8962c3 100644 --- a/public/app/features/explore/LogsVolumePanel.test.tsx +++ b/public/app/features/explore/LogsVolumePanel.test.tsx @@ -21,8 +21,6 @@ function renderPanel(logsVolumeData?: DataQueryResponse) { width={100} onUpdateTimeRange={() => {}} logsVolumeData={logsVolumeData} - logLinesBasedData={undefined} - logLinesBasedDataVisibleRange={undefined} onLoadLogsVolume={() => {}} onHiddenSeriesChanged={() => null} eventBus={new EventBusSrv()} @@ -31,11 +29,6 @@ function renderPanel(logsVolumeData?: DataQueryResponse) { } describe('LogsVolumePanel', () => { - it('shows loading message', () => { - renderPanel({ state: LoadingState.Loading, error: undefined, data: [] }); - expect(screen.getByText('Log volume is loading...')).toBeInTheDocument(); - }); - it('shows no volume data', () => { renderPanel({ state: LoadingState.Done, error: undefined, data: [] }); expect(screen.getByText('No volume data.')).toBeInTheDocument(); @@ -46,25 +39,6 @@ describe('LogsVolumePanel', () => { expect(screen.getByText('ExploreGraph')).toBeInTheDocument(); }); - it('shows short warning message', () => { - renderPanel({ state: LoadingState.Error, error: { data: { message: 'Test error message' } }, data: [] }); - expect(screen.getByText('Failed to load log volume for this query')).toBeInTheDocument(); - expect(screen.getByText('Test error message')).toBeInTheDocument(); - }); - - it('shows long warning message', () => { - // we make a long message - const messagePart = 'One two three four five six seven eight nine ten.'; - const message = messagePart + ' ' + messagePart + ' ' + messagePart; - - renderPanel({ state: LoadingState.Error, error: { data: { message } }, data: [] }); - expect(screen.getByText('Failed to load log volume for this query')).toBeInTheDocument(); - expect(screen.queryByText(message)).not.toBeInTheDocument(); - const button = screen.getByText('Show details'); - button.click(); - expect(screen.getByText(message)).toBeInTheDocument(); - }); - it('does not show the panel when there is no volume data', () => { renderPanel(undefined); expect(screen.queryByText('Log volume')).not.toBeInTheDocument(); diff --git a/public/app/features/explore/LogsVolumePanel.tsx b/public/app/features/explore/LogsVolumePanel.tsx index 4abdc0c5df1..d3f8c6deee1 100644 --- a/public/app/features/explore/LogsVolumePanel.tsx +++ b/public/app/features/explore/LogsVolumePanel.tsx @@ -4,23 +4,22 @@ import React from 'react'; import { AbsoluteTimeRange, DataQueryResponse, - GrafanaTheme2, LoadingState, SplitOpen, TimeZone, EventBus, - LogsVolumeType, + isLogsVolumeLimited, + getLogsVolumeAbsoluteRange, + GrafanaTheme2, + getLogsVolumeDataSourceInfo, } from '@grafana/data'; -import { Button, Collapse, Icon, InlineField, Tooltip, TooltipDisplayMode, useStyles2, useTheme2 } from '@grafana/ui'; +import { Icon, Tooltip, TooltipDisplayMode, useStyles2, useTheme2 } from '@grafana/ui'; import { ExploreGraph } from './Graph/ExploreGraph'; -import { SupplementaryResultError } from './SupplementaryResultError'; type Props = { logsVolumeData: DataQueryResponse | undefined; absoluteRange: AbsoluteTimeRange; - logLinesBasedData: DataQueryResponse | undefined; - logLinesBasedDataVisibleRange: AbsoluteTimeRange | undefined; timeZone: TimeZone; splitOpen: SplitOpen; width: number; @@ -31,7 +30,7 @@ type Props = { }; export function LogsVolumePanel(props: Props) { - const { width, timeZone, splitOpen, onUpdateTimeRange, onLoadLogsVolume, onHiddenSeriesChanged } = props; + const { width, timeZone, splitOpen, onUpdateTimeRange, onHiddenSeriesChanged } = props; const theme = useTheme2(); const styles = useStyles2(getStyles); const spacing = parseInt(theme.spacing(2).slice(0, -2), 10); @@ -42,17 +41,24 @@ export function LogsVolumePanel(props: Props) { } const logsVolumeData = props.logsVolumeData; - const range = logsVolumeData.data[0]?.meta?.custom?.absoluteRange || props.absoluteRange; - if (logsVolumeData.error !== undefined) { - return ; + const logsVolumeInfo = getLogsVolumeDataSourceInfo(logsVolumeData?.data); + let extraInfo = logsVolumeInfo ? `${logsVolumeInfo.refId} (${logsVolumeInfo.name})` : ''; + + if (isLogsVolumeLimited(logsVolumeData.data)) { + extraInfo = [ + extraInfo, + 'This datasource does not support full-range histograms. The graph below is based on the logs seen in the response.', + ].join('. '); } + const range = isLogsVolumeLimited(logsVolumeData.data) + ? getLogsVolumeAbsoluteRange(logsVolumeData.data, props.absoluteRange) + : props.absoluteRange; + let LogsVolumePanelContent; - if (logsVolumeData?.state === LoadingState.Loading) { - LogsVolumePanelContent = Log volume is loading...; - } else if (logsVolumeData?.data) { + if (logsVolumeData?.data) { if (logsVolumeData.data.length > 0) { LogsVolumePanelContent = ( {extraInfo}; - if (zoomRatio !== undefined && zoomRatio < 1) { - extraInfo = ( - -