PanelQueryRunner: Return previous processed (transform+field config) series for loading state (#67768)

* Return same series for loading state

* Fix shared query issue

* include structureRev

* heatmap should depend on the series, not the wrapper

* fix more panels

* keep config for comparison

* fieldConfig.fieldConfig!

* cleanup

* cmon

---------

Co-authored-by: Leon Sorokin <leeoniya@gmail.com>
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

.betterer.results

@@ -5594,8 +5594,7 @@ exports[`better eslint`] = {
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
       [0, 0, 0, "Do not use any type assertions.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "3"]
-      [0, 0, 0, "Do not use any type assertions.", "4"]
     ],
     "public/app/plugins/panel/heatmap/palettes.ts:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"],

public/app/features/query/state/PanelQueryRunner.ts

@@ -1,5 +1,5 @@
 import { cloneDeep } from 'lodash';
-import { MonoTypeOperatorFunction, Observable, of, ReplaySubject, Unsubscribable } from 'rxjs';
+import { Observable, of, ReplaySubject, Unsubscribable } from 'rxjs';
 import { map, mergeMap } from 'rxjs/operators';
 
 import {
@@ -26,6 +26,7 @@ import {
   toDataFrame,
   transformDataFrame,
   preProcessPanelData,
+  ApplyFieldOverrideOptions,
 } from '@grafana/data';
 import { getTemplateSrv, toDataQueryError } from '@grafana/runtime';
 import { ExpressionDatasourceRef } from '@grafana/runtime/src/utils/DataSourceWithBackend';
@@ -91,7 +92,9 @@ export class PanelQueryRunner {
   getData(options: GetDataOptions): Observable<PanelData> {
     const { withFieldConfig, withTransforms } = options;
     let structureRev = 1;
-    let lastData: DataFrame[] = [];
+    let lastFieldConfig: ApplyFieldOverrideOptions | undefined = undefined;
+    let lastProcessedFrames: DataFrame[] = [];
+    let lastRawFrames: DataFrame[] = [];
     let isFirstPacket = true;
     let lastConfigRev = -1;
 
@@ -105,97 +108,103 @@ export class PanelQueryRunner {
     }
 
     return this.subject.pipe(
-      this.getTransformationsStream(withTransforms),
+      mergeMap((data: PanelData) => {
-      map((data: PanelData) => {
+        let fieldConfig = this.dataConfigSource.getFieldOverrideOptions();
-        let processedData = data;
-        let streamingPacketWithSameSchema = false;
 
-        if (withFieldConfig && data.series?.length) {
+        if (data.series === lastRawFrames && lastFieldConfig?.fieldConfig === fieldConfig?.fieldConfig) {
-          if (lastConfigRev === this.dataConfigSource.configRev) {
+          return of({ ...data, structureRev, series: lastProcessedFrames });
-            const streamingDataFrame = data.series.find((data) => isStreamingDataFrame(data)) as
+        }
-              | StreamingDataFrame
+
-              | undefined;
+        lastFieldConfig = fieldConfig;
+        lastRawFrames = data.series;
+        let dataWithTransforms = of(data);
+
+        if (withTransforms) {
+          dataWithTransforms = this.applyTransformations(data);
+        }
+
+        return dataWithTransforms.pipe(
+          map((data: PanelData) => {
+            let processedData = data;
+            let streamingPacketWithSameSchema = false;
+
+            if (withFieldConfig && data.series?.length) {
+              if (lastConfigRev === this.dataConfigSource.configRev) {
+                const streamingDataFrame = data.series.find((data) => isStreamingDataFrame(data)) as
+                  | StreamingDataFrame
+                  | undefined;
+
+                if (
+                  streamingDataFrame &&
+                  !streamingDataFrame.packetInfo.schemaChanged &&
+                  // TODO: remove the condition below after fixing
+                  // https://github.com/grafana/grafana/pull/41492#issuecomment-970281430
+                  lastProcessedFrames[0].fields.length === streamingDataFrame.fields.length
+                ) {
+                  processedData = {
+                    ...processedData,
+                    series: lastProcessedFrames.map((frame, frameIndex) => ({
+                      ...frame,
+                      length: data.series[frameIndex].length,
+                      fields: frame.fields.map((field, fieldIndex) => ({
+                        ...field,
+                        values: data.series[frameIndex].fields[fieldIndex].values,
+                        state: {
+                          ...field.state,
+                          calcs: undefined,
+                          range: undefined,
+                        },
+                      })),
+                    })),
+                  };
+
+                  streamingPacketWithSameSchema = true;
+                }
+              }
+
+              if (fieldConfig != null && (isFirstPacket || !streamingPacketWithSameSchema)) {
+                lastConfigRev = this.dataConfigSource.configRev!;
+                processedData = {
+                  ...processedData,
+                  series: applyFieldOverrides({
+                    timeZone: data.request?.timezone ?? 'browser',
+                    data: processedData.series,
+                    ...fieldConfig!,
+                  }),
+                };
+                isFirstPacket = false;
+              }
+            }
 
             if (
-              streamingDataFrame &&
+              !streamingPacketWithSameSchema &&
-              !streamingDataFrame.packetInfo.schemaChanged &&
+              !compareArrayValues(lastProcessedFrames, processedData.series, compareDataFrameStructures)
-              // TODO: remove the condition below after fixing
-              // https://github.com/grafana/grafana/pull/41492#issuecomment-970281430
-              lastData[0].fields.length === streamingDataFrame.fields.length
             ) {
-              processedData = {
+              structureRev++;
-                ...processedData,
-                series: lastData.map((frame, frameIndex) => ({
-                  ...frame,
-                  length: data.series[frameIndex].length,
-                  fields: frame.fields.map((field, fieldIndex) => ({
-                    ...field,
-                    values: data.series[frameIndex].fields[fieldIndex].values,
-                    state: {
-                      ...field.state,
-                      calcs: undefined,
-                      range: undefined,
-                    },
-                  })),
-                })),
-              };
-
-              streamingPacketWithSameSchema = true;
             }
-          }
 
-          // Apply field defaults and overrides
+            lastProcessedFrames = processedData.series;
-          let fieldConfig = this.dataConfigSource.getFieldOverrideOptions();
 
-          if (fieldConfig != null && (isFirstPacket || !streamingPacketWithSameSchema)) {
+            return { ...processedData, structureRev };
-            lastConfigRev = this.dataConfigSource.configRev!;
+          })
-            processedData = {
+        );
-              ...processedData,
-              series: applyFieldOverrides({
-                timeZone: data.request?.timezone ?? 'browser',
-                data: processedData.series,
-                ...fieldConfig!,
-              }),
-            };
-            isFirstPacket = false;
-          }
-        }
-
-        if (
-          !streamingPacketWithSameSchema &&
-          !compareArrayValues(lastData, processedData.series, compareDataFrameStructures)
-        ) {
-          structureRev++;
-        }
-        lastData = processedData.series;
-
-        return { ...processedData, structureRev };
       })
     );
   }
 
-  private getTransformationsStream = (withTransforms: boolean): MonoTypeOperatorFunction<PanelData> => {
+  private applyTransformations(data: PanelData): Observable<PanelData> {
-    return (inputStream) =>
+    const transformations = this.dataConfigSource.getTransformations();
-      inputStream.pipe(
-        mergeMap((data) => {
-          if (!withTransforms) {
-            return of(data);
-          }
 
-          const transformations = this.dataConfigSource.getTransformations();
+    if (!transformations || transformations.length === 0) {
+      return of(data);
+    }
 
-          if (!transformations || transformations.length === 0) {
+    const ctx: DataTransformContext = {
-            return of(data);
+      interpolate: (v: string) => getTemplateSrv().replace(v, data?.request?.scopedVars),
-          }
+    };
 
-          const ctx: DataTransformContext = {
+    return transformDataFrame(transformations, data.series, ctx).pipe(map((series) => ({ ...data, series })));
-            interpolate: (v: string) => getTemplateSrv().replace(v, data?.request?.scopedVars),
+  }
-          };
-
-          return transformDataFrame(transformations, data.series, ctx).pipe(map((series) => ({ ...data, series })));
-        })
-      );
-  };
 
   async run(options: QueryRunnerOptions) {
     const {
@@ -216,7 +225,7 @@ export class PanelQueryRunner {
     } = options;
 
     if (isSharedDashboardQuery(datasource)) {
-      this.pipeToSubject(runSharedRequest(options, queries[0]), panelId);
+      this.pipeToSubject(runSharedRequest(options, queries[0]), panelId, true);
       return;
     }
 
@@ -284,7 +293,7 @@ export class PanelQueryRunner {
     }
   }
 
-  private pipeToSubject(observable: Observable<PanelData>, panelId?: number) {
+  private pipeToSubject(observable: Observable<PanelData>, panelId?: number, skipPreProcess = false) {
     if (this.subscription) {
       this.subscription.unsubscribe();
     }
@@ -299,7 +308,7 @@ export class PanelQueryRunner {
 
     this.subscription = panelData.subscribe({
       next: (data) => {
-        this.lastResult = preProcessPanelData(data, this.lastResult);
+        this.lastResult = skipPreProcess ? data : preProcessPanelData(data, this.lastResult);
         // Store preprocessed query results for applying overrides later on in the pipeline
         this.subject.next(this.lastResult);
       },

public/app/plugins/panel/barchart/BarChartPanel.tsx

@@ -98,7 +98,7 @@ export const BarChartPanel = ({ data, options, fieldConfig, width, height, timeZ
   const frame0Ref = useRef<DataFrame>();
   const colorByFieldRef = useRef<Field>();
 
-  const info = useMemo(() => prepareBarChartDisplayValues(data?.series, theme, options), [data, theme, options]);
+  const info = useMemo(() => prepareBarChartDisplayValues(data.series, theme, options), [data.series, theme, options]);
   const chartDisplay = 'viz' in info ? info : null;
 
   colorByFieldRef.current = chartDisplay?.colorByField;

public/app/plugins/panel/candlestick/CandlestickPanel.tsx

@@ -49,7 +49,7 @@ export const CandlestickPanel = ({
 
   const info = useMemo(() => {
     return prepareCandlestickFields(data.series, options, theme, timeRange);
-  }, [data, options, theme, timeRange]);
+  }, [data.series, options, theme, timeRange]);
 
   const { renderers, tweakScale, tweakAxis, shouldRenderPrice } = useMemo(() => {
     let tweakScale = (opts: ScaleProps, forField: Field) => opts;

public/app/plugins/panel/heatmap/HeatmapPanel.tsx

@@ -56,11 +56,11 @@ export const HeatmapPanel = ({
 
   const info = useMemo(() => {
     try {
-      return prepareHeatmapData(data, options, theme, getFieldLinksSupplier);
+      return prepareHeatmapData(data.series, data.annotations, options, theme, getFieldLinksSupplier);
     } catch (ex) {
       return { warning: `${ex}` };
     }
-  }, [data, options, theme, getFieldLinksSupplier]);
+  }, [data.series, data.annotations, options, theme, getFieldLinksSupplier]);
 
   const facets = useMemo(() => {
     let exemplarsXFacet: number[] = []; // "Time" field

public/app/plugins/panel/heatmap/fields.ts

@@ -8,7 +8,6 @@ import {
   GrafanaTheme2,
   LinkModel,
   outerJoinDataFrames,
-  PanelData,
   ValueFormatter,
   ValueLinkConfig,
 } from '@grafana/data';
@@ -56,17 +55,17 @@ export interface HeatmapData {
 }
 
 export function prepareHeatmapData(
-  data: PanelData,
+  frames: DataFrame[],
+  annotations: DataFrame[] | undefined,
   options: PanelOptions,
   theme: GrafanaTheme2,
   getFieldLinks?: (exemplars: DataFrame, field: Field) => (config: ValueLinkConfig) => Array<LinkModel<Field>>
 ): HeatmapData {
-  let frames = data.series;
   if (!frames?.length) {
     return {};
   }
 
-  const exemplars = data.annotations?.find((f) => f.name === 'exemplar');
+  const exemplars = annotations?.find((f) => f.name === 'exemplar');
 
   if (getFieldLinks) {
     exemplars?.fields.forEach((field, index) => {

public/app/plugins/panel/heatmap/module.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 
-import { FieldConfigProperty, FieldType, identityOverrideProcessor, PanelData, PanelPlugin } from '@grafana/data';
+import { FieldConfigProperty, FieldType, identityOverrideProcessor, PanelPlugin } from '@grafana/data';
 import { config } from '@grafana/runtime';
 import {
   AxisPlacement,
@@ -48,7 +48,7 @@ export const plugin = new PanelPlugin<PanelOptions, GraphFieldConfig>(HeatmapPan
     let isOrdinalY = false;
 
     try {
-      const v = prepareHeatmapData({ series: context.data } as PanelData, opts, config.theme2);
+      const v = prepareHeatmapData(context.data, undefined, opts, config.theme2);
       isOrdinalY = readHeatmapRowsCustomMeta(v.heatmap).yOrdinalDisplay != null;
     } catch {}
 

public/app/plugins/panel/heatmap/suggestions.ts

@@ -18,7 +18,7 @@ export class HeatmapSuggestionsSupplier {
       return;
     }
 
-    const info = prepareHeatmapData(builder.data, defaultPanelOptions, config.theme2);
+    const info = prepareHeatmapData(builder.data.series, undefined, defaultPanelOptions, config.theme2);
     if (!info || info.warning) {
       return;
     }

public/app/plugins/panel/histogram/HistogramPanel.tsx

@@ -13,7 +13,7 @@ export const HistogramPanel = ({ data, options, width, height }: Props) => {
   const theme = useTheme2();
 
   const histogram = useMemo(() => {
-    if (!data?.series?.length) {
+    if (!data.series.length) {
       return undefined;
     }
 

public/app/plugins/panel/state-timeline/StateTimelinePanel.tsx

@@ -71,8 +71,8 @@ export const StateTimelinePanel = ({
   };
 
   const { frames, warn } = useMemo(
-    () => prepareTimelineFields(data?.series, options.mergeValues ?? true, timeRange, theme),
+    () => prepareTimelineFields(data.series, options.mergeValues ?? true, timeRange, theme),
-    [data, options.mergeValues, timeRange, theme]
+    [data.series, options.mergeValues, timeRange, theme]
   );
 
   const legendItems = useMemo(

public/app/plugins/panel/status-history/StatusHistoryPanel.tsx

@@ -68,8 +68,8 @@ export const StatusHistoryPanel = ({
   };
 
   const { frames, warn } = useMemo(
-    () => prepareTimelineFields(data?.series, false, timeRange, theme),
+    () => prepareTimelineFields(data.series, false, timeRange, theme),
-    [data, timeRange, theme]
+    [data.series, timeRange, theme]
   );
 
   const legendItems = useMemo(

public/app/plugins/panel/timeseries/TimeSeriesPanel.tsx

@@ -38,7 +38,7 @@ export const TimeSeriesPanel = ({
     return getFieldLinksForExplore({ field, rowIndex, splitOpenFn: onSplitOpen, range: timeRange });
   };
 
-  const frames = useMemo(() => prepareGraphableFields(data.series, config.theme2, timeRange), [data, timeRange]);
+  const frames = useMemo(() => prepareGraphableFields(data.series, config.theme2, timeRange), [data.series, timeRange]);
   const timezones = useMemo(() => getTimezones(options.timezone, timeZone), [options.timezone, timeZone]);
   const suggestions = useMemo(() => {
     if (frames?.length && frames.every((df) => df.meta?.type === DataFrameType.TimeSeriesLong)) {

public/app/plugins/panel/trend/TrendPanel.tsx

@@ -66,7 +66,7 @@ export const TrendPanel = ({
     }
 
     return { frames: prepareGraphableFields(frames, config.theme2, undefined, xFieldIdx) };
-  }, [data, options.xField]);
+  }, [data.series, options.xField]);
 
   if (info.warning || !info.frames) {
     return (

public/app/plugins/panel/xychart/XYChartPanel2.tsx

@@ -90,7 +90,7 @@ export const XYChartPanel2 = (props: Props) => {
   useEffect(() => {
     if (oldOptions !== props.options || oldData?.structureRev !== props.data.structureRev) {
       initSeries();
-    } else if (oldData !== props.data) {
+    } else if (oldData?.series !== props.data.series) {
       initFacets();
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps