Organise transformations

packages/grafana-data/src/index.ts

@@ -1,4 +1,5 @@
-export * from './utils/index';
-export * from './types/index';
+export * from './utils';
+export * from './types';
 export * from './vector';
 export * from './dataframe';
+export * from './transformations';

packages/grafana-data/src/transformations/index.ts

@@ -0,0 +1,8 @@
+export * from './matchers/ids';
+export * from './transformers/ids';
+export * from './transformersRegistry';
+export * from './matchers/matchers';
+export * from './transformersRegistry';
+export * from './transformDataFrame';
+export { FilterFieldsByNameTransformerOptions } from './transformers/filterByName';
+export { ReduceTransformerOptions } from './transformers/reduce';

packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.test.ts

@@ -1,7 +1,7 @@
 import { FieldType } from '../../types/dataFrame';
 import { fieldMatchers } from './matchers';
 import { FieldMatcherID } from './ids';
-import { toDataFrame } from '../processDataFrame';
+import { toDataFrame } from '../../utils/processDataFrame';
 
 export const simpleSeriesWithTypes = toDataFrame({
   fields: [

packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.ts

@@ -1,6 +1,6 @@
 import { Field, FieldType } from '../../types/dataFrame';
-import { FieldMatcherInfo } from './matchers';
 import { FieldMatcherID } from './ids';
+import { FieldMatcherInfo } from '../../types/transformations';
 
 // General Field matcher
 const fieldTypeMacher: FieldMatcherInfo<FieldType> = {

packages/grafana-data/src/transformations/matchers/matchers.ts

@@ -1,27 +1,16 @@
-import { Field, DataFrame } from '../../types/dataFrame';
-import { Registry, RegistryItemWithOptions } from '../registry';
-
-export type FieldMatcher = (field: Field) => boolean;
-export type FrameMatcher = (frame: DataFrame) => boolean;
-
-export interface FieldMatcherInfo<TOptions = any> extends RegistryItemWithOptions<TOptions> {
-  get: (options: TOptions) => FieldMatcher;
-}
-
-export interface FrameMatcherInfo<TOptions = any> extends RegistryItemWithOptions<TOptions> {
-  get: (options: TOptions) => FrameMatcher;
-}
-
-export interface MatcherConfig<TOptions = any> {
-  id: string;
-  options?: TOptions;
-}
-
 // Load the Buildtin matchers
 import { getFieldPredicateMatchers, getFramePredicateMatchers } from './predicates';
 import { getFieldNameMatchers, getFrameNameMatchers } from './nameMatcher';
 import { getFieldTypeMatchers } from './fieldTypeMatcher';
 import { getRefIdMatchers } from './refIdMatcher';
+import {
+  FieldMatcherInfo,
+  MatcherConfig,
+  FrameMatcherInfo,
+  FieldMatcher,
+  FrameMatcher,
+} from '../../types/transformations';
+import { Registry } from '../../utils/registry';
 
 export const fieldMatchers = new Registry<FieldMatcherInfo>(() => {
   return [

packages/grafana-data/src/transformations/matchers/nameMatcher.test.ts

@@ -1,6 +1,6 @@
 import { getFieldMatcher } from './matchers';
 import { FieldMatcherID } from './ids';
-import { toDataFrame } from '../processDataFrame';
+import { toDataFrame } from '../../utils/processDataFrame';
 
 describe('Field Name Matcher', () => {
   it('Match all with wildcard regex', () => {

packages/grafana-data/src/transformations/matchers/nameMatcher.ts

@@ -1,7 +1,7 @@
 import { Field, DataFrame } from '../../types/dataFrame';
-import { FieldMatcherInfo, FrameMatcherInfo } from './matchers';
 import { FieldMatcherID, FrameMatcherID } from './ids';
-import { stringToJsRegex } from '../string';
+import { FieldMatcherInfo, FrameMatcherInfo } from '../../types/transformations';
+import { stringToJsRegex } from '../../utils/string';
 
 // General Field matcher
 const fieldNameMacher: FieldMatcherInfo<string> = {

packages/grafana-data/src/transformations/matchers/predicates.test.ts

@@ -1,7 +1,8 @@
 import { FieldType } from '../../types/dataFrame';
-import { MatcherConfig, fieldMatchers } from './matchers';
+import { fieldMatchers } from './matchers';
 import { simpleSeriesWithTypes } from './fieldTypeMatcher.test';
 import { FieldMatcherID, MatcherID } from './ids';
+import { MatcherConfig } from '../../types/transformations';
 
 const matchesNumberConfig: MatcherConfig = {
   id: FieldMatcherID.byType,

packages/grafana-data/src/transformations/matchers/predicates.ts

@@ -1,14 +1,7 @@
 import { Field, DataFrame } from '../../types/dataFrame';
 import { MatcherID } from './ids';
-import {
-  FrameMatcherInfo,
-  FieldMatcherInfo,
-  MatcherConfig,
-  getFieldMatcher,
-  fieldMatchers,
-  getFrameMatchers,
-  frameMatchers,
-} from './matchers';
+import { getFieldMatcher, fieldMatchers, getFrameMatchers, frameMatchers } from './matchers';
+import { FieldMatcherInfo, MatcherConfig, FrameMatcherInfo } from '../../types/transformations';
 
 const anyFieldMatcher: FieldMatcherInfo<MatcherConfig[]> = {
   id: MatcherID.anyMatch,

packages/grafana-data/src/transformations/matchers/refIdMatcher.ts

@@ -1,6 +1,6 @@
 import { DataFrame } from '../../types/dataFrame';
-import { FrameMatcherInfo } from './matchers';
 import { FrameMatcherID } from './ids';
+import { FrameMatcherInfo } from '../../types/transformations';
 
 // General Field matcher
 const refIdMacher: FrameMatcherInfo<string> = {

packages/grafana-data/src/transformations/transformDataFrame.ts

@@ -0,0 +1,31 @@
+import { DataTransformerConfig } from '../types/transformations';
+import { DataFrame } from '../types/dataFrame';
+import { transformersRegistry } from './transformersRegistry';
+
+/**
+ * Apply configured transformations to the input data
+ */
+export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): DataFrame[] {
+  let processed = data;
+  for (const config of options) {
+    const info = transformersRegistry.get(config.id);
+    const transformer = info.transformer(config.options);
+    const after = transformer(processed);
+
+    // Add a key to the metadata if the data changed
+    if (after && after !== processed) {
+      for (const series of after) {
+        if (!series.meta) {
+          series.meta = {};
+        }
+        if (!series.meta.transformations) {
+          series.meta.transformations = [info.id];
+        } else {
+          series.meta.transformations = [...series.meta.transformations, info.id];
+        }
+      }
+      processed = after;
+    }
+  }
+  return processed;
+}

packages/grafana-data/src/transformations/transformers/append.test.ts

@@ -1,6 +1,7 @@
-import { transformDataFrame, dataTransformers } from './transformers';
 import { DataTransformerID } from './ids';
-import { toDataFrame } from '../processDataFrame';
+import { toDataFrame } from '../../utils/processDataFrame';
+import { transformDataFrame } from '../transformDataFrame';
+import { transformersRegistry } from '../transformersRegistry';
 
 const seriesAB = toDataFrame({
   columns: [{ text: 'A' }, { text: 'B' }],
@@ -24,7 +25,7 @@ describe('Append Transformer', () => {
       id: DataTransformerID.append,
       options: {},
     };
-    const x = dataTransformers.get(DataTransformerID.append);
+    const x = transformersRegistry.get(DataTransformerID.append);
     expect(x.id).toBe(cfg.id);
 
     const processed = transformDataFrame([cfg], [seriesAB, seriesBC])[0];

packages/grafana-data/src/transformations/transformers/append.ts

@@ -1,7 +1,7 @@
-import { DataTransformerInfo } from './transformers';
 import { DataFrame } from '../../types/dataFrame';
 import { DataTransformerID } from './ids';
 import { MutableDataFrame } from '../../dataframe/MutableDataFrame';
+import { DataTransformerInfo } from '../../types/transformations';
 
 export interface AppendOptions {}
 

packages/grafana-data/src/transformations/transformers/filter.test.ts

@@ -1,8 +1,8 @@
 import { FieldType } from '../../types/dataFrame';
-import { FieldMatcherID } from '../matchers/ids';
-import { transformDataFrame } from './transformers';
 import { DataTransformerID } from './ids';
-import { toDataFrame } from '../processDataFrame';
+import { toDataFrame } from '../../utils/processDataFrame';
+import { transformDataFrame } from '../transformDataFrame';
+import { FieldMatcherID } from '../matchers/ids';
 
 export const simpleSeriesWithTypes = toDataFrame({
   fields: [

packages/grafana-data/src/transformations/transformers/filter.ts

@@ -1,9 +1,9 @@
-import { DataTransformerInfo } from './transformers';
 import { noopTransformer } from './noop';
 import { DataFrame, Field } from '../../types/dataFrame';
-import { FieldMatcherID } from '../matchers/ids';
 import { DataTransformerID } from './ids';
-import { MatcherConfig, getFieldMatcher, getFrameMatchers } from '../matchers/matchers';
+import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
+import { FieldMatcherID } from '../matchers/ids';
+import { getFieldMatcher, getFrameMatchers } from '../matchers/matchers';
 
 export interface FilterOptions {
   include?: MatcherConfig;

packages/grafana-data/src/transformations/transformers/filterByName.test.ts

@@ -1,6 +1,7 @@
-import { toDataFrame, transformDataFrame } from '../index';
+import { toDataFrame } from '../../utils/index';
 import { FieldType } from '../../index';
 import { DataTransformerID } from './ids';
+import { transformDataFrame } from '../transformDataFrame';
 
 export const seriesWithNamesToMatch = toDataFrame({
   fields: [

packages/grafana-data/src/transformations/transformers/filterByName.ts

@@ -1,7 +1,7 @@
-import { DataTransformerInfo } from './transformers';
-import { FieldMatcherID } from '../matchers/ids';
 import { DataTransformerID } from './ids';
 import { filterFieldsTransformer, FilterOptions } from './filter';
+import { DataTransformerInfo } from '../../types/transformations';
+import { FieldMatcherID } from '../matchers/ids';
 
 export interface FilterFieldsByNameTransformerOptions {
   include?: string;

packages/grafana-data/src/transformations/transformers/noop.ts

@@ -1,6 +1,6 @@
-import { DataTransformerInfo } from './transformers';
 import { DataTransformerID } from './ids';
 import { DataFrame } from '../../types/dataFrame';
+import { DataTransformerInfo } from '../../types/transformations';
 
 export interface NoopTransformerOptions {
   include?: string;

packages/grafana-data/src/transformations/transformers/reduce.test.ts

@@ -1,7 +1,7 @@
-import { transformDataFrame } from './transformers';
-import { ReducerID } from '../fieldReducer';
+import { ReducerID } from '../../utils/fieldReducer';
 import { DataTransformerID } from './ids';
-import { toDataFrame, toDataFrameDTO } from '../processDataFrame';
+import { toDataFrame, toDataFrameDTO } from '../../utils/processDataFrame';
+import { transformDataFrame } from '../transformDataFrame';
 
 const seriesWithValues = toDataFrame({
   fields: [

packages/grafana-data/src/transformations/transformers/reduce.ts

@@ -1,12 +1,12 @@
-import { DataTransformerInfo } from './transformers';
-import { DataFrame, FieldType, Field } from '../../types/dataFrame';
-import { MatcherConfig, getFieldMatcher } from '../matchers/matchers';
-import { alwaysFieldMatcher } from '../matchers/predicates';
 import { DataTransformerID } from './ids';
-import { ReducerID, fieldReducers, reduceField } from '../fieldReducer';
-import { KeyValue } from '../../types/data';
-import { guessFieldTypeForField } from '../processDataFrame';
+import { MatcherConfig, DataTransformerInfo } from '../../types/transformations';
+import { ReducerID, fieldReducers, reduceField } from '../../utils/fieldReducer';
+import { alwaysFieldMatcher } from '../matchers/predicates';
+import { DataFrame, Field, FieldType } from '../../types/dataFrame';
 import { ArrayVector } from '../../vector/ArrayVector';
+import { KeyValue } from '../../types/data';
+import { guessFieldTypeForField } from '../../utils/processDataFrame';
+import { getFieldMatcher } from '../matchers/matchers';
 
 export interface ReduceTransformerOptions {
   reducers: ReducerID[];

packages/grafana-data/src/transformations/transformersRegistry.test.ts

@@ -1,13 +1,13 @@
-import { DataTransformerID } from './ids';
-import { dataTransformers } from './transformers';
-import { toDataFrame } from '../processDataFrame';
-import { ReducerID } from '../fieldReducer';
-import { DataFrameView } from '../../dataframe/DataFrameView';
+import { DataTransformerID } from './transformers/ids';
+import { transformersRegistry } from './transformersRegistry';
+import { toDataFrame } from '../utils/processDataFrame';
+import { ReducerID } from '../utils/fieldReducer';
+import { DataFrameView } from '../dataframe/DataFrameView';
 
 describe('Transformers', () => {
   it('should load all transformeres', () => {
     for (const name of Object.keys(DataTransformerID)) {
-      const calc = dataTransformers.get(name);
+      const calc = transformersRegistry.get(name);
       expect(calc.id).toBe(name);
     }
   });
@@ -20,7 +20,7 @@ describe('Transformers', () => {
   });
 
   it('should use fluent API', () => {
-    const results = dataTransformers.reduce([seriesWithValues], {
+    const results = transformersRegistry.reduce([seriesWithValues], {
       reducers: [ReducerID.first],
     });
     expect(results.length).toBe(1);

packages/grafana-data/src/transformations/transformersRegistry.ts

@@ -0,0 +1,43 @@
+import { DataFrame } from '../types/dataFrame';
+import { Registry } from '../utils/registry';
+// Initalize the Registry
+
+import { appendTransformer, AppendOptions } from './transformers/append';
+import { reduceTransformer, ReduceTransformerOptions } from './transformers/reduce';
+import { filterFieldsTransformer, filterFramesTransformer } from './transformers/filter';
+import { filterFieldsByNameTransformer, FilterFieldsByNameTransformerOptions } from './transformers/filterByName';
+import { noopTransformer } from './transformers/noop';
+import { DataTransformerInfo } from '../types/transformations';
+
+/**
+ * Registry of transformation options that can be driven by
+ * stored configuration files.
+ */
+class TransformerRegistry extends Registry<DataTransformerInfo> {
+  // ------------------------------------------------------------
+  // Nacent options for more functional programming
+  // The API to these functions should change to match the actual
+  // needs of people trying to use it.
+  //  filterFields|Frames is left off since it is likely easier to
+  //  support with `frames.filter( f => {...} )`
+  // ------------------------------------------------------------
+
+  append(data: DataFrame[], options?: AppendOptions): DataFrame | undefined {
+    return appendTransformer.transformer(options || appendTransformer.defaultOptions)(data)[0];
+  }
+
+  reduce(data: DataFrame[], options: ReduceTransformerOptions): DataFrame[] {
+    return reduceTransformer.transformer(options)(data);
+  }
+}
+
+export const transformersRegistry = new TransformerRegistry(() => [
+  noopTransformer,
+  filterFieldsTransformer,
+  filterFieldsByNameTransformer,
+  filterFramesTransformer,
+  appendTransformer,
+  reduceTransformer,
+]);
+
+export { ReduceTransformerOptions, FilterFieldsByNameTransformerOptions };

packages/grafana-data/src/types/index.ts

@@ -11,3 +11,4 @@ export * from './valueMapping';
 export * from './displayValue';
 export * from './graph';
 export * from './ScopedVars';
+export * from './transformations';

packages/grafana-data/src/types/transformations.ts

@@ -0,0 +1,32 @@
+import { DataFrame, Field } from './dataFrame';
+import { RegistryItemWithOptions } from '../utils/registry';
+
+/**
+ * Immutable data transformation
+ */
+export type DataTransformer = (data: DataFrame[]) => DataFrame[];
+
+export interface DataTransformerInfo<TOptions = any> extends RegistryItemWithOptions {
+  transformer: (options: TOptions) => DataTransformer;
+}
+
+export interface DataTransformerConfig<TOptions = any> {
+  id: string;
+  options: TOptions;
+}
+
+export type FieldMatcher = (field: Field) => boolean;
+export type FrameMatcher = (frame: DataFrame) => boolean;
+
+export interface FieldMatcherInfo<TOptions = any> extends RegistryItemWithOptions<TOptions> {
+  get: (options: TOptions) => FieldMatcher;
+}
+
+export interface FrameMatcherInfo<TOptions = any> extends RegistryItemWithOptions<TOptions> {
+  get: (options: TOptions) => FrameMatcher;
+}
+
+export interface MatcherConfig<TOptions = any> {
+  id: string;
+  options?: TOptions;
+}

packages/grafana-data/src/utils/index.ts

@@ -19,8 +19,3 @@ export { getMappedValue } from './valueMappings';
 import * as dateMath from './datemath';
 import * as rangeUtil from './rangeutil';
 export { dateMath, rangeUtil };
-
-export * from './matchers/ids';
-export * from './matchers/matchers';
-export * from './transformers/ids';
-export * from './transformers/transformers';

packages/grafana-data/src/utils/transformers/transformers.ts

@@ -1,85 +0,0 @@
-import { DataFrame } from '../../types/dataFrame';
-import { Registry, RegistryItemWithOptions } from '../registry';
-
-/**
- * Immutable data transformation
- */
-export type DataTransformer = (data: DataFrame[]) => DataFrame[];
-
-export interface DataTransformerInfo<TOptions = any> extends RegistryItemWithOptions {
-  transformer: (options: TOptions) => DataTransformer;
-}
-
-export interface DataTransformerConfig<TOptions = any> {
-  id: string;
-  options: TOptions;
-}
-
-/**
- * Apply configured transformations to the input data
- */
-export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): DataFrame[] {
-  let processed = data;
-  for (const config of options) {
-    const info = dataTransformers.get(config.id);
-    const transformer = info.transformer(config.options);
-    const after = transformer(processed);
-
-    // Add a key to the metadata if the data changed
-    if (after && after !== processed) {
-      for (const series of after) {
-        if (!series.meta) {
-          series.meta = {};
-        }
-        if (!series.meta.transformations) {
-          series.meta.transformations = [info.id];
-        } else {
-          series.meta.transformations = [...series.meta.transformations, info.id];
-        }
-      }
-      processed = after;
-    }
-  }
-  return processed;
-}
-
-// Initalize the Registry
-
-import { appendTransformer, AppendOptions } from './append';
-import { reduceTransformer, ReduceTransformerOptions } from './reduce';
-import { filterFieldsTransformer, filterFramesTransformer } from './filter';
-import { filterFieldsByNameTransformer, FilterFieldsByNameTransformerOptions } from './filterByName';
-import { noopTransformer } from './noop';
-
-/**
- * Registry of transformation options that can be driven by
- * stored configuration files.
- */
-class TransformerRegistry extends Registry<DataTransformerInfo> {
-  // ------------------------------------------------------------
-  // Nacent options for more functional programming
-  // The API to these functions should change to match the actual
-  // needs of people trying to use it.
-  //  filterFields|Frames is left off since it is likely easier to
-  //  support with `frames.filter( f => {...} )`
-  // ------------------------------------------------------------
-
-  append(data: DataFrame[], options?: AppendOptions): DataFrame | undefined {
-    return appendTransformer.transformer(options || appendTransformer.defaultOptions)(data)[0];
-  }
-
-  reduce(data: DataFrame[], options: ReduceTransformerOptions): DataFrame[] {
-    return reduceTransformer.transformer(options)(data);
-  }
-}
-
-export const dataTransformers = new TransformerRegistry(() => [
-  noopTransformer,
-  filterFieldsTransformer,
-  filterFieldsByNameTransformer,
-  filterFramesTransformer,
-  appendTransformer,
-  reduceTransformer,
-]);
-
-export { ReduceTransformerOptions, FilterFieldsByNameTransformerOptions };

packages/grafana-ui/src/components/TransformersUI/FilterByNameTransformerEditor.tsx

@@ -1,5 +1,5 @@
 import React, { useContext } from 'react';
-import { FilterFieldsByNameTransformerOptions, DataTransformerID, dataTransformers, KeyValue } from '@grafana/data';
+import { FilterFieldsByNameTransformerOptions, DataTransformerID, transformersRegistry, KeyValue } from '@grafana/data';
 import { TransformerUIProps, TransformerUIRegistyItem } from './types';
 import { ThemeContext } from '../../themes/ThemeContext';
 import { css, cx } from 'emotion';
@@ -157,7 +157,7 @@ const FilterPill: React.FC<FilterPillProps> = ({ label, selected, onClick }) =>
 export const filterFieldsByNameTransformRegistryItem: TransformerUIRegistyItem<FilterFieldsByNameTransformerOptions> = {
   id: DataTransformerID.filterFieldsByName,
   component: FilterByNameTransformerEditor,
-  transformer: dataTransformers.get(DataTransformerID.filterFieldsByName),
+  transformer: transformersRegistry.get(DataTransformerID.filterFieldsByName),
   name: 'Filter by name',
   description: 'UI for filter by name transformation',
 };

packages/grafana-ui/src/components/TransformersUI/ReduceTransformerEditor.tsx

@@ -1,8 +1,7 @@
 import React from 'react';
 import { StatsPicker } from '../StatsPicker/StatsPicker';
-import { ReduceTransformerOptions, DataTransformerID, ReducerID } from '@grafana/data';
+import { ReduceTransformerOptions, DataTransformerID, ReducerID, transformersRegistry } from '@grafana/data';
 import { TransformerUIRegistyItem, TransformerUIProps } from './types';
-import { dataTransformers } from '@grafana/data';
 
 // TODO:  Minimal implementation, needs some <3
 export const ReduceTransformerEditor: React.FC<TransformerUIProps<ReduceTransformerOptions>> = ({
@@ -29,7 +28,7 @@ export const ReduceTransformerEditor: React.FC<TransformerUIProps<ReduceTransfor
 export const reduceTransformRegistryItem: TransformerUIRegistyItem<ReduceTransformerOptions> = {
   id: DataTransformerID.reduce,
   component: ReduceTransformerEditor,
-  transformer: dataTransformers.get(DataTransformerID.reduce),
+  transformer: transformersRegistry.get(DataTransformerID.reduce),
   name: 'Reduce',
   description: 'UI for reduce transformation',
 };