QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899)

* I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
2019-09-12 17:28:46 +02:00
parent 3742db720f
commit 140ecbcf79
49 changed files with 1620 additions and 1525 deletions
@@ -393,8 +393,8 @@ export class CircularDataFrame<T = any> extends MutableDataFrame<T> {
  constructor(options: CircularOptions) {
    super(undefined, (buffer?: any[]) => {
      return new CircularVector({
-        buffer,
        ...options,
+        buffer,
      });
    });
  }
@@ -101,6 +101,7 @@ export function reduceField(options: ReduceFieldOptions): FieldCalcs {

  // For now everything can use the standard stats
  let values = doStandardCalcs(field, ignoreNulls, nullAsZero);
+
  for (const reducer of queue) {
    if (!values.hasOwnProperty(reducer.id) && reducer.reduce) {
      values = {
@@ -109,10 +110,12 @@ export function reduceField(options: ReduceFieldOptions): FieldCalcs {
      };
    }
  }
+
  field.calcs = {
    ...field.calcs,
    ...values,
  };
+
  return values;
 }

@@ -238,16 +238,19 @@ export const toDataFrame = (data: any): DataFrame => {
    // This will convert the array values into Vectors
    return new MutableDataFrame(data as DataFrameDTO);
  }
+
  if (data.hasOwnProperty('datapoints')) {
    return convertTimeSeriesToDataFrame(data);
  }
+
  if (data.hasOwnProperty('data')) {
    return convertGraphSeriesToDataFrame(data);
  }
+
  if (data.hasOwnProperty('columns')) {
    return convertTableToDataFrame(data);
  }
-  // TODO, try to convert JSON/Array to table?
+
  console.warn('Can not convert', data);
  throw new Error('Unsupported data format');
 };
@@ -13,7 +13,7 @@ interface TransformationsEditorState {
 interface TransformationsEditorProps {
  onChange: (transformations: DataTransformerConfig[]) => void;
  transformations: DataTransformerConfig[];
-  getCurrentData: (applyTransformations?: boolean) => DataFrame[];
+  dataFrames: DataFrame[];
 }

 export class TransformationsEditor extends React.PureComponent<TransformationsEditorProps, TransformationsEditorState> {
@@ -46,9 +46,9 @@ export class TransformationsEditor extends React.PureComponent<TransformationsEd
  };

  renderTransformationEditors = () => {
-    const { transformations, getCurrentData } = this.props;
+    const { transformations, dataFrames } = this.props;
    const hasTransformations = transformations.length > 0;
-    const preTransformData = getCurrentData(false);
+    const preTransformData = dataFrames;

    if (!hasTransformations) {
      return undefined;
@@ -13,6 +13,7 @@ import {
 } from '@grafana/data';
 import { PluginMeta, GrafanaPlugin } from './plugin';
 import { PanelData } from './panel';
+import { Observable } from 'rxjs';

 // NOTE: this seems more general than just DataSource
 export interface DataSourcePluginOptionsEditorProps<TOptions> {
@@ -189,26 +190,9 @@ export abstract class DataSourceApi<
  init?: () => void;

  /**
-   * Query for data, and optionally stream results to an observer.
-   *
-   * Are you reading these docs aiming to execute a query?
-   * +-> If Yes, then consider using panelQueryRunner/State instead.  see:
-   *  * {@link https://github.com/grafana/grafana/blob/master/public/app/features/dashboard/state/PanelQueryRunner.ts PanelQueryRunner.ts}
-   *  * {@link https://github.com/grafana/grafana/blob/master/public/app/features/dashboard/state/PanelQueryState.ts PanelQueryState.ts}
-   *
-   * If you are implementing a simple request-response query,
-   * then you can ignore the `observer` entirely.
-   *
-   * When streaming behavior is required, the Promise can return at any time
-   * with empty or partial data in the response and optionally a state.
-   * NOTE: The data in this initial response will not be replaced with any
-   * data from subsequent events. {@see DataStreamState}
-   *
-   * The request object will be passed in each observer callback
-   * so the callback could assert that the correct events are streaming and
-   * unsubscribe if unexpected results are returned.
+   * Query for data, and optionally stream results
   */
-  abstract query(request: DataQueryRequest<TQuery>, observer?: DataStreamObserver): Promise<DataQueryResponse>;
+  abstract query(request: DataQueryRequest<TQuery>): Promise<DataQueryResponse> | Observable<DataQueryResponse>;

  /**
   * Test & verify datasource settings & connection details
@@ -397,6 +381,19 @@ export interface DataQueryResponse {
   * or a partial result set
   */
  data: DataQueryResponseData[];
+
+  /**
+   * When returning multiple partial responses or streams
+   * Use this key to inform Grafana how to combine the partial responses
+   * Multiple responses with same key are replaced (latest used)
+   */
+  key?: string;
+
+  /**
+   * Use this to control which state the response should have
+   * Defaults to LoadingState.Done if state is not defined
+   */
+  state?: LoadingState;
 }

 export interface DataQuery {
@@ -1,6 +1,6 @@
 import { ComponentClass, ComponentType } from 'react';
 import { LoadingState, DataFrame, TimeRange, TimeZone, ScopedVars } from '@grafana/data';
-import { DataQueryRequest, DataQueryError, LegacyResponseData } from './datasource';
+import { DataQueryRequest, DataQueryError } from './datasource';
 import { PluginMeta, GrafanaPlugin } from './plugin';

 export type InterpolateFunction = (value: string, scopedVars?: ScopedVars, format?: string | Function) => string;
@@ -16,9 +16,6 @@ export interface PanelData {
  series: DataFrame[];
  request?: DataQueryRequest;
  error?: DataQueryError;
-
-  // Data format expected by Angular panels
-  legacy?: LegacyResponseData[];
 }

 export interface PanelProps<T = any> {