Explore: Clear live logs (#64237)

* feat: add clear logs to explorer's live logs

* refactor live logs pause/resume updating logic

* add tests for clearing live logs

* rm unused imports

* move `onClear` live logs logic to redux

* move clear logs tests to `query.test`

* use utils `sortLogsRows` and Button's icon props

* rename `filterLogRows` and add `clearedAt` as an arg

* mv clearedAt type closer to live tailing items and add jsdoc

* mv `filterLogRowsByTime` to `/utils` and use it at `LiveLogs` component

* make `Exit live` button consistent with other actions

* use `sortLogRows` and fix timestamp by id on `makeLogs`

* change clear live logs from based on timestamp to index on logRows

* assign `null` to `clearedAtIndex` on first batch of logs live

* move `isFirstStreaming` implementation to `clearLogs` reducer

* fix `clearLogs` tests

* Update public/app/features/explore/state/query.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

---------

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

public/app/features/explore/LiveLogs.test.tsx

@@ -1,9 +1,10 @@
 import { render, screen } from '@testing-library/react';
 import React from 'react';
 
-import { LogLevel, LogRowModel, MutableDataFrame } from '@grafana/data';
+import { LogRowModel } from '@grafana/data';
 
 import { LiveLogsWithTheme } from './LiveLogs';
+import { makeLogs } from './__mocks__/makeLogs';
 
 const setup = (rows: LogRowModel[]) =>
   render(
@@ -13,36 +14,16 @@ const setup = (rows: LogRowModel[]) =>
       stopLive={() => {}}
       onPause={() => {}}
       onResume={() => {}}
+      onClear={() => {}}
+      clearedAtIndex={null}
       isPaused={true}
     />
   );
 
-const makeLog = (overrides: Partial<LogRowModel>): LogRowModel => {
-  const uid = overrides.uid || '1';
-  const entry = `log message ${uid}`;
-  return {
-    uid,
-    entryFieldIndex: 0,
-    rowIndex: 0,
-    dataFrame: new MutableDataFrame(),
-    logLevel: LogLevel.debug,
-    entry,
-    hasAnsi: false,
-    hasUnescapedContent: false,
-    labels: {},
-    raw: entry,
-    timeFromNow: '',
-    timeEpochMs: 1,
-    timeEpochNs: '1000000',
-    timeLocal: '',
-    timeUtc: '',
-    ...overrides,
-  };
-};
-
 describe('LiveLogs', () => {
   it('renders logs', () => {
-    setup([makeLog({ uid: '1' }), makeLog({ uid: '2' }), makeLog({ uid: '3' })]);
+    const logRows = makeLogs(3);
+    setup(logRows);
 
     expect(screen.getByRole('cell', { name: 'log message 1' })).toBeInTheDocument();
     expect(screen.getByRole('cell', { name: 'log message 2' })).toBeInTheDocument();
@@ -50,14 +31,19 @@ describe('LiveLogs', () => {
   });
 
   it('renders new logs only when not paused', () => {
-    const { rerender } = setup([makeLog({ uid: '1' }), makeLog({ uid: '2' }), makeLog({ uid: '3' })]);
+    const logRows = makeLogs(6);
+    const firstLogs = logRows.slice(0, 3);
+    const secondLogs = logRows.slice(3, 6);
+    const { rerender } = setup(firstLogs);
 
     rerender(
       <LiveLogsWithTheme
-        logRows={[makeLog({ uid: '4' }), makeLog({ uid: '5' }), makeLog({ uid: '6' })]}
+        logRows={secondLogs}
         timeZone={'utc'}
         stopLive={() => {}}
         onPause={() => {}}
+        onClear={() => {}}
+        clearedAtIndex={null}
         onResume={() => {}}
         isPaused={true}
       />
@@ -72,11 +58,13 @@ describe('LiveLogs', () => {
 
     rerender(
       <LiveLogsWithTheme
-        logRows={[makeLog({ uid: '4' }), makeLog({ uid: '5' }), makeLog({ uid: '6' })]}
+        logRows={secondLogs}
         timeZone={'utc'}
         stopLive={() => {}}
         onPause={() => {}}
         onResume={() => {}}
+        onClear={() => {}}
+        clearedAtIndex={null}
         isPaused={false}
       />
     );
@@ -87,11 +75,12 @@ describe('LiveLogs', () => {
   });
 
   it('renders ansi logs', () => {
-    setup([
+    const commonLog = makeLogs(1);
-      makeLog({ uid: '1' }),
+    const firstAnsiLog = makeLogs(1, { hasAnsi: true, raw: 'log message \u001B[31m2\u001B[0m', uid: '2' });
-      makeLog({ hasAnsi: true, raw: 'log message \u001B[31m2\u001B[0m', uid: '2' }),
+    const secondAnsiLog = makeLogs(1, { hasAnsi: true, raw: 'log message \u001B[33m3\u001B[0m', uid: '3' });
-      makeLog({ hasAnsi: true, raw: 'log message \u001B[33m3\u001B[0m', uid: '3' }),
+    const logRows = [...commonLog, ...firstAnsiLog, ...secondAnsiLog];
-    ]);
+
+    setup(logRows);
 
     expect(screen.getByRole('cell', { name: 'log message 1' })).toBeInTheDocument();
     expect(screen.getByRole('cell', { name: 'log message 2' })).toBeInTheDocument();

public/app/features/explore/LiveLogs.tsx

@@ -2,13 +2,15 @@ import { css, cx } from '@emotion/css';
 import React, { PureComponent } from 'react';
 import tinycolor from 'tinycolor2';
 
-import { LogRowModel, TimeZone, dateTimeFormat, GrafanaTheme2 } from '@grafana/data';
+import { LogRowModel, TimeZone, dateTimeFormat, GrafanaTheme2, LogsSortOrder } from '@grafana/data';
-import { Icon, Button, Themeable2, withTheme2 } from '@grafana/ui';
+import { Button, Themeable2, withTheme2 } from '@grafana/ui';
 
 import { LogMessageAnsi } from '../logs/components/LogMessageAnsi';
 import { getLogRowStyles } from '../logs/components/getLogRowStyles';
+import { sortLogRows } from '../logs/utils';
 
 import { ElapsedTime } from './ElapsedTime';
+import { filterLogRowsByIndex } from './state/utils';
 
 const getStyles = (theme: GrafanaTheme2) => ({
   logsRowsLive: css`
@@ -57,6 +59,8 @@ export interface Props extends Themeable2 {
   stopLive: () => void;
   onPause: () => void;
   onResume: () => void;
+  onClear: () => void;
+  clearedAtIndex: number | null;
   isPaused: boolean;
 }
 
@@ -76,16 +80,22 @@ class LiveLogs extends PureComponent<Props, State> {
   }
 
   static getDerivedStateFromProps(nextProps: Props, state: State) {
-    if (!nextProps.isPaused) {
+    if (nextProps.isPaused && nextProps.clearedAtIndex) {
       return {
-        // We update what we show only if not paused. We keep any background subscriptions running and keep updating
+        logRowsToRender: filterLogRowsByIndex(nextProps.clearedAtIndex, state.logRowsToRender),
-        // our state, but we do not show the updates, this allows us start again showing correct result after resuming
-        // without creating a gap in the log results.
-        logRowsToRender: nextProps.logRows,
       };
-    } else {
+    }
+
+    if (nextProps.isPaused) {
       return null;
     }
+
+    return {
+      // We update what we show only if not paused. We keep any background subscriptions running and keep updating
+      // our state, but we do not show the updates, this allows us start again showing correct result after resuming
+      // without creating a gap in the log results.
+      logRowsToRender: nextProps.logRows,
+    };
   }
 
   /**
@@ -107,13 +117,13 @@ class LiveLogs extends PureComponent<Props, State> {
     let { logRowsToRender: rowsToRender = [] } = this.state;
     if (!isPaused) {
       // A perf optimisation here. Show just 100 rows when streaming and full length when the streaming is paused.
-      rowsToRender = rowsToRender.slice(-100);
+      rowsToRender = sortLogRows(rowsToRender, LogsSortOrder.Ascending).slice(-100);
     }
     return rowsToRender;
   };
 
   render() {
-    const { theme, timeZone, onPause, onResume, isPaused } = this.props;
+    const { theme, timeZone, onPause, onResume, onClear, isPaused } = this.props;
     const styles = getStyles(theme);
     const { logsRow, logsRowLocalTime, logsRowMessage } = getLogRowStyles(theme);
 
@@ -147,20 +157,26 @@ class LiveLogs extends PureComponent<Props, State> {
           </tbody>
         </table>
         <div className={styles.logsRowsIndicator}>
-          <Button variant="secondary" onClick={isPaused ? onResume : onPause} className={styles.button}>
+          <Button
-            <Icon name={isPaused ? 'play' : 'pause'} />
+            icon={isPaused ? 'play' : 'pause'}
-            &nbsp;
+            variant="secondary"
+            onClick={isPaused ? onResume : onPause}
+            className={styles.button}
+          >
             {isPaused ? 'Resume' : 'Pause'}
           </Button>
-          <Button variant="secondary" onClick={this.props.stopLive} className={styles.button}>
+          <Button icon="trash-alt" variant="secondary" onClick={onClear} className={styles.button}>
-            <Icon name="square-shape" size="lg" type="mono" />
+            Clear logs
-            &nbsp; Exit live mode
           </Button>
-          {isPaused || (
+          <Button icon="square-shape" variant="secondary" onClick={this.props.stopLive} className={styles.button}>
-            <span>
+            Exit live mode
-              Last line received: <ElapsedTime resetKey={this.props.logRows} humanize={true} /> ago
+          </Button>
-            </span>
+          {isPaused ||
-          )}
+            (this.rowsToRender().length > 0 && (
+              <span>
+                Last line received: <ElapsedTime resetKey={this.props.logRows} humanize={true} /> ago
+              </span>
+            ))}
         </div>
       </div>
     );

public/app/features/explore/LogsContainer.tsx

@@ -137,6 +137,8 @@ class LogsContainer extends PureComponent<LogsContainerProps> {
                   isPaused={this.props.isPaused}
                   onPause={controls.pause}
                   onResume={controls.resume}
+                  onClear={controls.clear}
+                  clearedAtIndex={this.props.clearedAtIndex}
                 />
               )}
             </LiveTailControls>
@@ -195,6 +197,7 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string }
     datasourceInstance,
     isLive,
     isPaused,
+    clearedAtIndex,
     range,
     absoluteRange,
     supplementaryQueries,
@@ -214,6 +217,7 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string }
     datasourceInstance,
     isLive,
     isPaused,
+    clearedAtIndex,
     range,
     absoluteRange,
     logsVolume,

public/app/features/explore/__mocks__/makeLogs.ts

@@ -0,0 +1,32 @@
+import { MutableDataFrame, LogLevel, LogRowModel, LogsSortOrder } from '@grafana/data';
+import { sortLogRows } from 'app/features/logs/utils';
+export const makeLogs = (numberOfLogsToCreate: number, overrides?: Partial<LogRowModel>): LogRowModel[] => {
+  const array: LogRowModel[] = [];
+
+  for (let i = 0; i < numberOfLogsToCreate; i++) {
+    const uuid = (i + 1).toString();
+    const entry = `log message ${uuid}`;
+    const timeInMs = overrides?.timeEpochMs || new Date().getTime();
+
+    array.push({
+      uid: uuid,
+      entryFieldIndex: 0,
+      rowIndex: 0,
+      dataFrame: new MutableDataFrame(),
+      logLevel: LogLevel.debug,
+      entry,
+      hasAnsi: false,
+      hasUnescapedContent: false,
+      labels: {},
+      raw: entry,
+      timeFromNow: '',
+      timeEpochMs: timeInMs + i,
+      timeEpochNs: (timeInMs * 1000000 + i).toString(),
+      timeLocal: '',
+      timeUtc: '',
+      ...overrides,
+    });
+  }
+
+  return sortLogRows(array, LogsSortOrder.Ascending);
+};

public/app/features/explore/state/query.test.ts

@@ -22,6 +22,7 @@ import { ExploreId, ExploreItemState, StoreState, ThunkDispatch } from 'app/type
 import { reducerTester } from '../../../../test/core/redux/reducerTester';
 import { configureStore } from '../../../store/configureStore';
 import { setTimeSrv, TimeSrv } from '../../dashboard/services/TimeSrv';
+import { makeLogs } from '../__mocks__/makeLogs';
 import { supplementaryQueryTypes } from '../utils/supplementaryQueries';
 
 import { createDefaultInitialState } from './helpers';
@@ -41,6 +42,9 @@ import {
   setSupplementaryQueryEnabled,
   addQueryRow,
   cleanSupplementaryQueryDataProviderAction,
+  clearLogs,
+  queryStreamUpdatedAction,
+  QueryEndedPayload,
   changeQueries,
 } from './query';
 import * as actions from './query';
@@ -924,4 +928,77 @@ describe('reducer', () => {
       ).toBeUndefined();
     });
   });
+  describe('clear live logs', () => {
+    it('should clear current log rows', async () => {
+      const logRows = makeLogs(10);
+
+      const { dispatch, getState }: { dispatch: ThunkDispatch; getState: () => StoreState } = configureStore({
+        ...defaultInitialState,
+        explore: {
+          [ExploreId.left]: {
+            ...defaultInitialState.explore[ExploreId.left],
+            queryResponse: {
+              state: LoadingState.Streaming,
+            },
+            logsResult: {
+              hasUniqueLabels: false,
+              rows: logRows,
+            },
+          },
+        },
+      } as unknown as Partial<StoreState>);
+      expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(logRows.length);
+
+      await dispatch(clearLogs({ exploreId: ExploreId.left }));
+
+      expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(0);
+      expect(getState().explore[ExploreId.left].clearedAtIndex).toBe(logRows.length - 1);
+    });
+
+    it('should filter new log rows', async () => {
+      const oldLogRows = makeLogs(10);
+      const newLogRows = makeLogs(5);
+      const allLogRows = [...oldLogRows, ...newLogRows];
+
+      const { dispatch, getState }: { dispatch: ThunkDispatch; getState: () => StoreState } = configureStore({
+        ...defaultInitialState,
+        explore: {
+          [ExploreId.left]: {
+            ...defaultInitialState.explore[ExploreId.left],
+            isLive: true,
+            queryResponse: {
+              state: LoadingState.Streaming,
+            },
+            logsResult: {
+              hasUniqueLabels: false,
+              rows: oldLogRows,
+            },
+          },
+        },
+      } as unknown as Partial<StoreState>);
+
+      expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(oldLogRows.length);
+
+      await dispatch(clearLogs({ exploreId: ExploreId.left }));
+      await dispatch(
+        queryStreamUpdatedAction({
+          exploreId: ExploreId.left,
+          response: {
+            request: true,
+            traceFrames: [],
+            nodeGraphFrames: [],
+            rawPrometheusFrames: [],
+            flameGraphFrames: [],
+            logsResult: {
+              hasUniqueLabels: false,
+              rows: allLogRows,
+            },
+          },
+        } as unknown as QueryEndedPayload)
+      );
+
+      expect(getState().explore[ExploreId.left].logsResult?.rows.length).toBe(newLogRows.length);
+      expect(getState().explore[ExploreId.left].clearedAtIndex).toBe(oldLogRows.length - 1);
+    });
+  });
 });

public/app/features/explore/state/query.ts

@@ -61,7 +61,7 @@ import {
 import { addHistoryItem, historyUpdatedAction, loadRichHistory } from './history';
 import { stateSave } from './main';
 import { updateTime } from './time';
-import { createCacheKey, getResultsFromCache } from './utils';
+import { createCacheKey, getResultsFromCache, filterLogRowsByIndex } from './utils';
 
 //
 // Actions and Payloads
@@ -191,6 +191,10 @@ export interface SetPausedStatePayload {
 }
 export const setPausedStateAction = createAction<SetPausedStatePayload>('explore/setPausedState');
 
+export interface ClearLogsPayload {
+  exploreId: ExploreId;
+}
+export const clearLogs = createAction<ClearLogsPayload>('explore/clearLogs');
 /**
  * Start a scan for more results using the given scanner.
  * @param exploreId Explore area
@@ -1089,6 +1093,41 @@ export const queryReducer = (state: ExploreItemState, action: AnyAction): Explor
     };
   }
 
+  if (clearLogs.match(action)) {
+    if (!state.logsResult) {
+      return {
+        ...state,
+        clearedAtIndex: null,
+      };
+    }
+
+    // When in loading state, clear logs and set clearedAtIndex as null.
+    // Initially loaded logs will be fully replaced by incoming streamed logs, which may have a different length.
+    if (state.queryResponse.state === LoadingState.Loading) {
+      return {
+        ...state,
+        clearedAtIndex: null,
+        logsResult: {
+          ...state.logsResult,
+          rows: [],
+        },
+      };
+    }
+
+    const lastItemIndex = state.clearedAtIndex
+      ? state.clearedAtIndex + state.logsResult.rows.length
+      : state.logsResult.rows.length - 1;
+
+    return {
+      ...state,
+      clearedAtIndex: lastItemIndex,
+      logsResult: {
+        ...state.logsResult,
+        rows: [],
+      },
+    };
+  }
+
   return state;
 };
 
@@ -1113,7 +1152,6 @@ const getCorrelations = () => {
     }
   });
 };
-
 export const processQueryResponse = (
   state: ExploreItemState,
   action: PayloadAction<QueryEndedPayload>
@@ -1169,7 +1207,10 @@ export const processQueryResponse = (
     graphResult,
     tableResult,
     rawPrometheusResult,
-    logsResult,
+    logsResult:
+      state.isLive && logsResult
+        ? { ...logsResult, rows: filterLogRowsByIndex(state.clearedAtIndex, logsResult.rows) }
+        : logsResult,
     loading: loadingState === LoadingState.Loading || loadingState === LoadingState.Streaming,
     showLogs: !!logsResult,
     showMetrics: !!graphResult,
@@ -1178,5 +1219,6 @@ export const processQueryResponse = (
     showNodeGraph: !!nodeGraphFrames.length,
     showRawPrometheus: !!rawPrometheusFrames.length,
     showFlameGraph: !!flameGraphFrames.length,
+    clearedAtIndex: state.isLive ? state.clearedAtIndex : null,
   };
 };

public/app/features/explore/state/utils.ts

@@ -8,6 +8,7 @@ import {
   getDefaultTimeRange,
   HistoryItem,
   LoadingState,
+  LogRowModel,
   PanelData,
 } from '@grafana/data';
 import { DataSourceRef } from '@grafana/schema';
@@ -58,6 +59,7 @@ export const makeExplorePaneState = (): ExploreItemState => ({
   tableResult: null,
   graphResult: null,
   logsResult: null,
+  clearedAtIndex: null,
   rawPrometheusResult: null,
   eventBridge: null as unknown as EventBusExtended,
   cache: [],
@@ -158,3 +160,19 @@ export function getResultsFromCache(
   const cacheValue = cacheIdx >= 0 ? cache[cacheIdx].value : undefined;
   return cacheValue;
 }
+
+export const filterLogRowsByIndex = (
+  clearedAtIndex: ExploreItemState['clearedAtIndex'],
+  logRows?: LogRowModel[]
+): LogRowModel[] => {
+  if (!logRows) {
+    return [];
+  }
+
+  if (clearedAtIndex) {
+    const filteredRows = logRows.slice(clearedAtIndex + 1);
+    return filteredRows;
+  }
+
+  return logRows;
+};

public/app/features/explore/useLiveTailControls.ts

@@ -5,7 +5,7 @@ import { useDispatch } from 'app/types';
 
 import { ExploreId } from '../../types';
 
-import { setPausedStateAction, runQueries } from './state/query';
+import { setPausedStateAction, runQueries, clearLogs } from './state/query';
 import { changeRefreshInterval } from './state/time';
 
 /**
@@ -38,11 +38,16 @@ export function useLiveTailControls(exploreId: ExploreId) {
     dispatch(changeRefreshInterval(exploreId, RefreshPicker.liveOption.value));
   }, [exploreId, dispatch]);
 
+  const clear = useCallback(() => {
+    dispatch(clearLogs({ exploreId }));
+  }, [exploreId, dispatch]);
+
   return {
     pause,
     resume,
     stop,
     start,
+    clear,
   };
 }
 

public/app/types/explore.ts

@@ -175,6 +175,12 @@ export interface ExploreItemState {
    */
   isPaused: boolean;
 
+  /**
+   * Index of the last item in the list of logs
+   * when the live tailing views gets cleared.
+   */
+  clearedAtIndex: number | null;
+
   querySubscription?: Unsubscribable;
 
   queryResponse: ExplorePanelData;