From 3a013cbe480c4ebebe9e911df7122a34c8a90b4f Mon Sep 17 00:00:00 2001 From: Abdul Date: Thu, 20 Apr 2023 05:21:28 -0300 Subject: [PATCH] Explore: Clear live logs (#64237) * feat: add clear logs to explorer's live logs * refactor live logs pause/resume updating logic * add tests for clearing live logs * rm unused imports * move `onClear` live logs logic to redux * move clear logs tests to `query.test` * use utils `sortLogsRows` and Button's icon props * rename `filterLogRows` and add `clearedAt` as an arg * mv clearedAt type closer to live tailing items and add jsdoc * mv `filterLogRowsByTime` to `/utils` and use it at `LiveLogs` component * make `Exit live` button consistent with other actions * use `sortLogRows` and fix timestamp by id on `makeLogs` * change clear live logs from based on timestamp to index on logRows * assign `null` to `clearedAtIndex` on first batch of logs live * move `isFirstStreaming` implementation to `clearLogs` reducer * fix `clearLogs` tests * Update public/app/features/explore/state/query.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --------- Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --- public/app/features/explore/LiveLogs.test.tsx | 55 ++++++------- public/app/features/explore/LiveLogs.tsx | 58 +++++++++----- public/app/features/explore/LogsContainer.tsx | 4 + .../features/explore/__mocks__/makeLogs.ts | 32 ++++++++ .../app/features/explore/state/query.test.ts | 77 +++++++++++++++++++ public/app/features/explore/state/query.ts | 48 +++++++++++- public/app/features/explore/state/utils.ts | 18 +++++ .../features/explore/useLiveTailControls.ts | 7 +- public/app/types/explore.ts | 6 ++ 9 files changed, 247 insertions(+), 58 deletions(-) create mode 100644 public/app/features/explore/__mocks__/makeLogs.ts diff --git a/public/app/features/explore/LiveLogs.test.tsx b/public/app/features/explore/LiveLogs.test.tsx index ceebb895113..854897b1361 100644 --- a/public/app/features/explore/LiveLogs.test.tsx +++ b/public/app/features/explore/LiveLogs.test.tsx @@ -1,9 +1,10 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; -import { LogLevel, LogRowModel, MutableDataFrame } from '@grafana/data'; +import { LogRowModel } from '@grafana/data'; import { LiveLogsWithTheme } from './LiveLogs'; +import { makeLogs } from './__mocks__/makeLogs'; const setup = (rows: LogRowModel[]) => render( @@ -13,36 +14,16 @@ const setup = (rows: LogRowModel[]) => stopLive={() => {}} onPause={() => {}} onResume={() => {}} + onClear={() => {}} + clearedAtIndex={null} isPaused={true} /> ); -const makeLog = (overrides: Partial): LogRowModel => { - const uid = overrides.uid || '1'; - const entry = `log message ${uid}`; - return { - uid, - entryFieldIndex: 0, - rowIndex: 0, - dataFrame: new MutableDataFrame(), - logLevel: LogLevel.debug, - entry, - hasAnsi: false, - hasUnescapedContent: false, - labels: {}, - raw: entry, - timeFromNow: '', - timeEpochMs: 1, - timeEpochNs: '1000000', - timeLocal: '', - timeUtc: '', - ...overrides, - }; -}; - describe('LiveLogs', () => { it('renders logs', () => { - setup([makeLog({ uid: '1' }), makeLog({ uid: '2' }), makeLog({ uid: '3' })]); + const logRows = makeLogs(3); + setup(logRows); expect(screen.getByRole('cell', { name: 'log message 1' })).toBeInTheDocument(); expect(screen.getByRole('cell', { name: 'log message 2' })).toBeInTheDocument(); @@ -50,14 +31,19 @@ describe('LiveLogs', () => { }); it('renders new logs only when not paused', () => { - const { rerender } = setup([makeLog({ uid: '1' }), makeLog({ uid: '2' }), makeLog({ uid: '3' })]); + const logRows = makeLogs(6); + const firstLogs = logRows.slice(0, 3); + const secondLogs = logRows.slice(3, 6); + const { rerender } = setup(firstLogs); rerender( {}} onPause={() => {}} + onClear={() => {}} + clearedAtIndex={null} onResume={() => {}} isPaused={true} /> @@ -72,11 +58,13 @@ describe('LiveLogs', () => { rerender( {}} onPause={() => {}} onResume={() => {}} + onClear={() => {}} + clearedAtIndex={null} isPaused={false} /> ); @@ -87,11 +75,12 @@ describe('LiveLogs', () => { }); it('renders ansi logs', () => { - setup([ - makeLog({ uid: '1' }), - makeLog({ hasAnsi: true, raw: 'log message \u001B[31m2\u001B[0m', uid: '2' }), - makeLog({ hasAnsi: true, raw: 'log message \u001B[33m3\u001B[0m', uid: '3' }), - ]); + const commonLog = makeLogs(1); + const firstAnsiLog = makeLogs(1, { hasAnsi: true, raw: 'log message \u001B[31m2\u001B[0m', uid: '2' }); + const secondAnsiLog = makeLogs(1, { hasAnsi: true, raw: 'log message \u001B[33m3\u001B[0m', uid: '3' }); + const logRows = [...commonLog, ...firstAnsiLog, ...secondAnsiLog]; + + setup(logRows); expect(screen.getByRole('cell', { name: 'log message 1' })).toBeInTheDocument(); expect(screen.getByRole('cell', { name: 'log message 2' })).toBeInTheDocument(); diff --git a/public/app/features/explore/LiveLogs.tsx b/public/app/features/explore/LiveLogs.tsx index d5cfb36ef03..5d258946562 100644 --- a/public/app/features/explore/LiveLogs.tsx +++ b/public/app/features/explore/LiveLogs.tsx @@ -2,13 +2,15 @@ import { css, cx } from '@emotion/css'; import React, { PureComponent } from 'react'; import tinycolor from 'tinycolor2'; -import { LogRowModel, TimeZone, dateTimeFormat, GrafanaTheme2 } from '@grafana/data'; -import { Icon, Button, Themeable2, withTheme2 } from '@grafana/ui'; +import { LogRowModel, TimeZone, dateTimeFormat, GrafanaTheme2, LogsSortOrder } from '@grafana/data'; +import { Button, Themeable2, withTheme2 } from '@grafana/ui'; import { LogMessageAnsi } from '../logs/components/LogMessageAnsi'; import { getLogRowStyles } from '../logs/components/getLogRowStyles'; +import { sortLogRows } from '../logs/utils'; import { ElapsedTime } from './ElapsedTime'; +import { filterLogRowsByIndex } from './state/utils'; const getStyles = (theme: GrafanaTheme2) => ({ logsRowsLive: css` @@ -57,6 +59,8 @@ export interface Props extends Themeable2 { stopLive: () => void; onPause: () => void; onResume: () => void; + onClear: () => void; + clearedAtIndex: number | null; isPaused: boolean; } @@ -76,16 +80,22 @@ class LiveLogs extends PureComponent { } static getDerivedStateFromProps(nextProps: Props, state: State) { - if (!nextProps.isPaused) { + if (nextProps.isPaused && nextProps.clearedAtIndex) { return { - // We update what we show only if not paused. We keep any background subscriptions running and keep updating - // our state, but we do not show the updates, this allows us start again showing correct result after resuming - // without creating a gap in the log results. - logRowsToRender: nextProps.logRows, + logRowsToRender: filterLogRowsByIndex(nextProps.clearedAtIndex, state.logRowsToRender), }; - } else { + } + + if (nextProps.isPaused) { return null; } + + return { + // We update what we show only if not paused. We keep any background subscriptions running and keep updating + // our state, but we do not show the updates, this allows us start again showing correct result after resuming + // without creating a gap in the log results. + logRowsToRender: nextProps.logRows, + }; } /** @@ -107,13 +117,13 @@ class LiveLogs extends PureComponent { let { logRowsToRender: rowsToRender = [] } = this.state; if (!isPaused) { // A perf optimisation here. Show just 100 rows when streaming and full length when the streaming is paused. - rowsToRender = rowsToRender.slice(-100); + rowsToRender = sortLogRows(rowsToRender, LogsSortOrder.Ascending).slice(-100); } return rowsToRender; }; render() { - const { theme, timeZone, onPause, onResume, isPaused } = this.props; + const { theme, timeZone, onPause, onResume, onClear, isPaused } = this.props; const styles = getStyles(theme); const { logsRow, logsRowLocalTime, logsRowMessage } = getLogRowStyles(theme); @@ -147,20 +157,26 @@ class LiveLogs extends PureComponent {
- - - {isPaused || ( - - Last line received: ago - - )} + + {isPaused || + (this.rowsToRender().length > 0 && ( + + Last line received: ago + + ))}
); diff --git a/public/app/features/explore/LogsContainer.tsx b/public/app/features/explore/LogsContainer.tsx index eb98c9584b5..24211ee973b 100644 --- a/public/app/features/explore/LogsContainer.tsx +++ b/public/app/features/explore/LogsContainer.tsx @@ -137,6 +137,8 @@ class LogsContainer extends PureComponent { isPaused={this.props.isPaused} onPause={controls.pause} onResume={controls.resume} + onClear={controls.clear} + clearedAtIndex={this.props.clearedAtIndex} /> )} @@ -195,6 +197,7 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string } datasourceInstance, isLive, isPaused, + clearedAtIndex, range, absoluteRange, supplementaryQueries, @@ -214,6 +217,7 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string } datasourceInstance, isLive, isPaused, + clearedAtIndex, range, absoluteRange, logsVolume, diff --git a/public/app/features/explore/__mocks__/makeLogs.ts b/public/app/features/explore/__mocks__/makeLogs.ts new file mode 100644 index 00000000000..0aaf4d9f254 --- /dev/null +++ b/public/app/features/explore/__mocks__/makeLogs.ts @@ -0,0 +1,32 @@ +import { MutableDataFrame, LogLevel, LogRowModel, LogsSortOrder } from '@grafana/data'; +import { sortLogRows } from 'app/features/logs/utils'; +export const makeLogs = (numberOfLogsToCreate: number, overrides?: Partial): LogRowModel[] => { + const array: LogRowModel[] = []; + + for (let i = 0; i < numberOfLogsToCreate; i++) { + const uuid = (i + 1).toString(); + const entry = `log message ${uuid}`; + const timeInMs = overrides?.timeEpochMs || new Date().getTime(); + + array.push({ + uid: uuid, + entryFieldIndex: 0, + rowIndex: 0, + dataFrame: new MutableDataFrame(), + logLevel: LogLevel.debug, + entry, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + raw: entry, + timeFromNow: '', + timeEpochMs: timeInMs + i, + timeEpochNs: (timeInMs * 1000000 + i).toString(), + timeLocal: '', + timeUtc: '', + ...overrides, + }); + } + + return sortLogRows(array, LogsSortOrder.Ascending); +}; diff --git a/public/app/features/explore/state/query.test.ts b/public/app/features/explore/state/query.test.ts index d6a00955f8c..4e26abac682 100644 --- a/public/app/features/explore/state/query.test.ts +++ b/public/app/features/explore/state/query.test.ts @@ -22,6 +22,7 @@ import { ExploreId, ExploreItemState, StoreState, ThunkDispatch } from 'app/type import { reducerTester } from '../../../../test/core/redux/reducerTester'; import { configureStore } from '../../../store/configureStore'; import { setTimeSrv, TimeSrv } from '../../dashboard/services/TimeSrv'; +import { makeLogs } from '../__mocks__/makeLogs'; import { supplementaryQueryTypes } from '../utils/supplementaryQueries'; import { createDefaultInitialState } from './helpers'; @@ -41,6 +42,9 @@ import { setSupplementaryQueryEnabled, addQueryRow, cleanSupplementaryQueryDataProviderAction, + clearLogs, + queryStreamUpdatedAction, + QueryEndedPayload, changeQueries, } from './query'; import * as actions from './query'; @@ -924,4 +928,77 @@ describe('reducer', () => { ).toBeUndefined(); }); }); + describe('clear live logs', () => { + it('should clear current log rows', async () => { + const logRows = makeLogs(10); + + const { dispatch, getState }: { dispatch: ThunkDispatch; getState: () => StoreState } = configureStore({ + ...defaultInitialState, + explore: { + [ExploreId.left]: { + ...defaultInitialState.explore[ExploreId.left], + queryResponse: { + state: LoadingState.Streaming, + }, + logsResult: { + hasUniqueLabels: false, + rows: logRows, + }, + }, + }, + } as unknown as Partial); + expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(logRows.length); + + await dispatch(clearLogs({ exploreId: ExploreId.left })); + + expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(0); + expect(getState().explore[ExploreId.left].clearedAtIndex).toBe(logRows.length - 1); + }); + + it('should filter new log rows', async () => { + const oldLogRows = makeLogs(10); + const newLogRows = makeLogs(5); + const allLogRows = [...oldLogRows, ...newLogRows]; + + const { dispatch, getState }: { dispatch: ThunkDispatch; getState: () => StoreState } = configureStore({ + ...defaultInitialState, + explore: { + [ExploreId.left]: { + ...defaultInitialState.explore[ExploreId.left], + isLive: true, + queryResponse: { + state: LoadingState.Streaming, + }, + logsResult: { + hasUniqueLabels: false, + rows: oldLogRows, + }, + }, + }, + } as unknown as Partial); + + expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(oldLogRows.length); + + await dispatch(clearLogs({ exploreId: ExploreId.left })); + await dispatch( + queryStreamUpdatedAction({ + exploreId: ExploreId.left, + response: { + request: true, + traceFrames: [], + nodeGraphFrames: [], + rawPrometheusFrames: [], + flameGraphFrames: [], + logsResult: { + hasUniqueLabels: false, + rows: allLogRows, + }, + }, + } as unknown as QueryEndedPayload) + ); + + expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(newLogRows.length); + expect(getState().explore[ExploreId.left].clearedAtIndex).toBe(oldLogRows.length - 1); + }); + }); }); diff --git a/public/app/features/explore/state/query.ts b/public/app/features/explore/state/query.ts index 822f8fbc970..1b9ecacc758 100644 --- a/public/app/features/explore/state/query.ts +++ b/public/app/features/explore/state/query.ts @@ -61,7 +61,7 @@ import { import { addHistoryItem, historyUpdatedAction, loadRichHistory } from './history'; import { stateSave } from './main'; import { updateTime } from './time'; -import { createCacheKey, getResultsFromCache } from './utils'; +import { createCacheKey, getResultsFromCache, filterLogRowsByIndex } from './utils'; // // Actions and Payloads @@ -191,6 +191,10 @@ export interface SetPausedStatePayload { } export const setPausedStateAction = createAction('explore/setPausedState'); +export interface ClearLogsPayload { + exploreId: ExploreId; +} +export const clearLogs = createAction('explore/clearLogs'); /** * Start a scan for more results using the given scanner. * @param exploreId Explore area @@ -1089,6 +1093,41 @@ export const queryReducer = (state: ExploreItemState, action: AnyAction): Explor }; } + if (clearLogs.match(action)) { + if (!state.logsResult) { + return { + ...state, + clearedAtIndex: null, + }; + } + + // When in loading state, clear logs and set clearedAtIndex as null. + // Initially loaded logs will be fully replaced by incoming streamed logs, which may have a different length. + if (state.queryResponse.state === LoadingState.Loading) { + return { + ...state, + clearedAtIndex: null, + logsResult: { + ...state.logsResult, + rows: [], + }, + }; + } + + const lastItemIndex = state.clearedAtIndex + ? state.clearedAtIndex + state.logsResult.rows.length + : state.logsResult.rows.length - 1; + + return { + ...state, + clearedAtIndex: lastItemIndex, + logsResult: { + ...state.logsResult, + rows: [], + }, + }; + } + return state; }; @@ -1113,7 +1152,6 @@ const getCorrelations = () => { } }); }; - export const processQueryResponse = ( state: ExploreItemState, action: PayloadAction @@ -1169,7 +1207,10 @@ export const processQueryResponse = ( graphResult, tableResult, rawPrometheusResult, - logsResult, + logsResult: + state.isLive && logsResult + ? { ...logsResult, rows: filterLogRowsByIndex(state.clearedAtIndex, logsResult.rows) } + : logsResult, loading: loadingState === LoadingState.Loading || loadingState === LoadingState.Streaming, showLogs: !!logsResult, showMetrics: !!graphResult, @@ -1178,5 +1219,6 @@ export const processQueryResponse = ( showNodeGraph: !!nodeGraphFrames.length, showRawPrometheus: !!rawPrometheusFrames.length, showFlameGraph: !!flameGraphFrames.length, + clearedAtIndex: state.isLive ? state.clearedAtIndex : null, }; }; diff --git a/public/app/features/explore/state/utils.ts b/public/app/features/explore/state/utils.ts index 868c82df53a..3dd4d90efbf 100644 --- a/public/app/features/explore/state/utils.ts +++ b/public/app/features/explore/state/utils.ts @@ -8,6 +8,7 @@ import { getDefaultTimeRange, HistoryItem, LoadingState, + LogRowModel, PanelData, } from '@grafana/data'; import { DataSourceRef } from '@grafana/schema'; @@ -58,6 +59,7 @@ export const makeExplorePaneState = (): ExploreItemState => ({ tableResult: null, graphResult: null, logsResult: null, + clearedAtIndex: null, rawPrometheusResult: null, eventBridge: null as unknown as EventBusExtended, cache: [], @@ -158,3 +160,19 @@ export function getResultsFromCache( const cacheValue = cacheIdx >= 0 ? cache[cacheIdx].value : undefined; return cacheValue; } + +export const filterLogRowsByIndex = ( + clearedAtIndex: ExploreItemState['clearedAtIndex'], + logRows?: LogRowModel[] +): LogRowModel[] => { + if (!logRows) { + return []; + } + + if (clearedAtIndex) { + const filteredRows = logRows.slice(clearedAtIndex + 1); + return filteredRows; + } + + return logRows; +}; diff --git a/public/app/features/explore/useLiveTailControls.ts b/public/app/features/explore/useLiveTailControls.ts index 6b9ed004d93..652e5aaf9a9 100644 --- a/public/app/features/explore/useLiveTailControls.ts +++ b/public/app/features/explore/useLiveTailControls.ts @@ -5,7 +5,7 @@ import { useDispatch } from 'app/types'; import { ExploreId } from '../../types'; -import { setPausedStateAction, runQueries } from './state/query'; +import { setPausedStateAction, runQueries, clearLogs } from './state/query'; import { changeRefreshInterval } from './state/time'; /** @@ -38,11 +38,16 @@ export function useLiveTailControls(exploreId: ExploreId) { dispatch(changeRefreshInterval(exploreId, RefreshPicker.liveOption.value)); }, [exploreId, dispatch]); + const clear = useCallback(() => { + dispatch(clearLogs({ exploreId })); + }, [exploreId, dispatch]); + return { pause, resume, stop, start, + clear, }; } diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index 5d97a69b4d0..4c24e6d5e4f 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -175,6 +175,12 @@ export interface ExploreItemState { */ isPaused: boolean; + /** + * Index of the last item in the list of logs + * when the live tailing views gets cleared. + */ + clearedAtIndex: number | null; + querySubscription?: Unsubscribable; queryResponse: ExplorePanelData;