From 963544ffe6a69096cdc292895bf6d1cb82f39e83 Mon Sep 17 00:00:00 2001 From: nikki-kiga <42276368+nikki-kiga@users.noreply.github.com> Date: Wed, 27 Oct 2021 20:20:56 -0700 Subject: [PATCH] Transformations: Add gazetteer transformation (#40967) --- .../src/transformations/transformers/ids.ts | 1 + .../FieldLookupTransformerEditor.tsx | 91 ++++++++++++++ .../lookupGazetteer/fieldLookup.test.ts | 115 ++++++++++++++++++ .../lookupGazetteer/fieldLookup.ts | 69 +++++++++++ public/app/core/utils/standardTransformers.ts | 2 + .../geomap/editor/GazetteerPathEditor.tsx | 15 ++- public/app/plugins/panel/geomap/types.ts | 5 +- 7 files changed, 293 insertions(+), 5 deletions(-) create mode 100644 public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx create mode 100644 public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts create mode 100644 public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index 67f9a8cdc4c..b2aafaa5cfe 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -27,4 +27,5 @@ export enum DataTransformerID { rowsToFields = 'rowsToFields', prepareTimeSeries = 'prepareTimeSeries', convertFieldType = 'convertFieldType', + fieldLookup = 'fieldLookup', } diff --git a/public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx b/public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx new file mode 100644 index 00000000000..ba2528d22dd --- /dev/null +++ b/public/app/core/components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor.tsx @@ -0,0 +1,91 @@ +import React, { useCallback } from 'react'; +import { + DataTransformerID, + FieldNamePickerConfigSettings, + PluginState, + StandardEditorsRegistryItem, + TransformerRegistryItem, + TransformerUIProps, +} from '@grafana/data'; + +import { InlineField, InlineFieldRow } from '@grafana/ui'; +import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker'; +import { GazetteerPathEditor } from 'app/plugins/panel/geomap/editor/GazetteerPathEditor'; +import { GazetteerPathEditorConfigSettings } from 'app/plugins/panel/geomap/types'; +import { FieldLookupOptions, fieldLookupTransformer } from './fieldLookup'; +import { FieldType } from '../../../../../../packages/grafana-data/src'; + +const fieldNamePickerSettings: StandardEditorsRegistryItem = { + settings: { + width: 30, + filter: (f) => f.type === FieldType.string, + placeholderText: 'Select text field', + noFieldsMessage: 'No text fields found', + }, + name: '', + id: '', + editor: () => null, +}; + +const fieldLookupSettings: StandardEditorsRegistryItem = { + settings: {}, +} as any; + +export const FieldLookupTransformerEditor: React.FC> = ({ + input, + options, + onChange, +}) => { + const onPickLookupField = useCallback( + (value: string | undefined) => { + onChange({ + ...options, + lookupField: value, + }); + }, + [onChange, options] + ); + + const onPickGazetteer = useCallback( + (value: string | undefined) => { + onChange({ + ...options, + gazetteer: value, + }); + }, + [onChange, options] + ); + return ( +
+ + + + + + + + + + +
+ ); +}; + +export const fieldLookupTransformRegistryItem: TransformerRegistryItem = { + id: DataTransformerID.fieldLookup, + editor: FieldLookupTransformerEditor, + transformation: fieldLookupTransformer, + name: 'Field lookup', + description: `Use a field value to lookup additional fields from an external source. This current supports spatial data, but will eventuall support more formats`, + state: PluginState.alpha, +}; diff --git a/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts b/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts new file mode 100644 index 00000000000..41428b035f4 --- /dev/null +++ b/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.test.ts @@ -0,0 +1,115 @@ +import { FieldMatcherID, fieldMatchers, FieldType } from '@grafana/data'; +import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame'; +import { DataTransformerID } from '@grafana/data/src/transformations/transformers/ids'; +import { Gazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer'; +import { addFieldsFromGazetteer } from './fieldLookup'; + +describe('Lookup gazetteer', () => { + it('adds lat/lon based on string field', async () => { + const cfg = { + id: DataTransformerID.fieldLookup, + options: { + lookupField: 'location', + gazetteer: 'public/gazetteer/usa-states.json', + }, + }; + const data = toDataFrame({ + name: 'locations', + fields: [ + { name: 'location', type: FieldType.string, values: ['AL', 'AK', 'Arizona', 'Arkansas', 'Somewhere'] }, + { name: 'values', type: FieldType.number, values: [0, 10, 5, 1, 5] }, + ], + }); + + const matcher = fieldMatchers.get(FieldMatcherID.byName).get(cfg.options?.lookupField); + + const values = new Map() + .set('AL', { name: 'Alabama', id: 'AL', coords: [-80.891064, 12.448457] }) + .set('AK', { name: 'Arkansas', id: 'AK', coords: [-100.891064, 24.448457] }) + .set('AZ', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] }) + .set('Arizona', { name: 'Arizona', id: 'AZ', coords: [-111.891064, 33.448457] }); + + const gaz: Gazetteer = { + count: 3, + examples: () => ['AL', 'AK', 'AZ'], + find: (k) => { + let v = values.get(k); + if (!v && typeof k === 'string') { + v = values.get(k.toUpperCase()); + } + return v; + }, + path: 'public/gazetteer/usa-states.json', + }; + + expect(await addFieldsFromGazetteer([data], gaz, matcher)).toMatchInlineSnapshot(` + Array [ + Object { + "creator": [Function], + "fields": Array [ + Object { + "config": Object {}, + "name": "location", + "type": "string", + "values": Array [ + "AL", + "AK", + "Arizona", + "Arkansas", + "Somewhere", + ], + }, + Object { + "config": Object {}, + "name": "lon", + "type": "number", + "values": Array [ + -80.891064, + -100.891064, + -111.891064, + undefined, + undefined, + ], + }, + Object { + "config": Object {}, + "name": "lat", + "type": "number", + "values": Array [ + 12.448457, + 24.448457, + 33.448457, + undefined, + undefined, + ], + }, + Object { + "config": Object {}, + "name": "values", + "state": Object { + "displayName": "values", + }, + "type": "number", + "values": Array [ + 0, + 10, + 5, + 1, + 5, + ], + }, + ], + "first": Array [ + "AL", + "AK", + "Arizona", + "Arkansas", + "Somewhere", + ], + "length": 5, + "name": "locations", + }, + ] + `); + }); +}); diff --git a/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts b/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts new file mode 100644 index 00000000000..9c42db918c3 --- /dev/null +++ b/public/app/core/components/TransformersUI/lookupGazetteer/fieldLookup.ts @@ -0,0 +1,69 @@ +import { + ArrayVector, + DataFrame, + DataTransformerID, + Field, + FieldMatcher, + FieldMatcherID, + fieldMatchers, + FieldType, + DataTransformerInfo, +} from '@grafana/data'; +import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from 'app/plugins/panel/geomap/gazetteer/gazetteer'; +import { mergeMap, from } from 'rxjs'; + +export interface FieldLookupOptions { + lookupField?: string; + gazetteer?: string; +} + +export const fieldLookupTransformer: DataTransformerInfo = { + id: DataTransformerID.fieldLookup, + name: 'Lookup fields from resource', + description: 'Retrieve matching data based on specified field', + defaultOptions: {}, + + operator: (options) => (source) => source.pipe(mergeMap((data) => from(doGazetteerXform(data, options)))), +}; + +async function doGazetteerXform(frames: DataFrame[], options: FieldLookupOptions): Promise { + const fieldMatches = fieldMatchers.get(FieldMatcherID.byName).get(options?.lookupField); + + const gaz = await getGazetteer(options?.gazetteer ?? COUNTRIES_GAZETTEER_PATH); + + return addFieldsFromGazetteer(frames, gaz, fieldMatches); +} + +export function addFieldsFromGazetteer(frames: DataFrame[], gaz: Gazetteer, matcher: FieldMatcher): DataFrame[] { + return frames.map((frame) => { + const fields: Field[] = []; + + for (const field of frame.fields) { + fields.push(field); + + //if the field matches + if (matcher(field, frame, frames)) { + const values = field.values.toArray(); + const lat = new Array(values.length); + const lon = new Array(values.length); + + //for each value find the corresponding value in the gazetteer + for (let v = 0; v < values.length; v++) { + const foundMatchingValue = gaz.find(values[v]); + + //for now start by adding lat and lon + if (foundMatchingValue && foundMatchingValue?.coords.length) { + lon[v] = foundMatchingValue.coords[0]; + lat[v] = foundMatchingValue.coords[1]; + } + } + fields.push({ name: 'lon', type: FieldType.number, values: new ArrayVector(lon), config: {} }); + fields.push({ name: 'lat', type: FieldType.number, values: new ArrayVector(lat), config: {} }); + } + } + return { + ...frame, + fields, + }; + }); +} diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts index 1e2002e4754..ca52df1a508 100644 --- a/public/app/core/utils/standardTransformers.ts +++ b/public/app/core/utils/standardTransformers.ts @@ -18,6 +18,7 @@ import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/ import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor'; import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor'; import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor'; +import { fieldLookupTransformRegistryItem } from '../components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor'; export const getStandardTransformers = (): Array> => { return [ @@ -40,5 +41,6 @@ export const getStandardTransformers = (): Array> = configFromQueryTransformRegistryItem, prepareTimeseriesTransformerRegistryItem, convertFieldTypeTransformRegistryItem, + fieldLookupTransformRegistryItem, ]; }; diff --git a/public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx b/public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx index dd75f7ab9a5..5c749da7ab4 100644 --- a/public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx +++ b/public/app/plugins/panel/geomap/editor/GazetteerPathEditor.tsx @@ -3,8 +3,9 @@ import { StandardEditorProps, SelectableValue, GrafanaTheme2 } from '@grafana/da import { Alert, Select, stylesFactory, useTheme2 } from '@grafana/ui'; import { COUNTRIES_GAZETTEER_PATH, Gazetteer, getGazetteer } from '../gazetteer/gazetteer'; import { css } from '@emotion/css'; +import { GazetteerPathEditorConfigSettings } from '../types'; -const paths: Array> = [ +const defaultPaths: Array> = [ { label: 'Countries', description: 'Lookup countries by name, two letter code, or three letter code', @@ -22,9 +23,15 @@ const paths: Array> = [ }, ]; -export const GazetteerPathEditor: FC> = ({ value, onChange, context }) => { +export const GazetteerPathEditor: FC> = ({ + value, + onChange, + context, + item, +}) => { const styles = getStyles(useTheme2()); const [gaz, setGaz] = useState(); + const settings = item.settings as any; useEffect(() => { async function fetchData() { @@ -35,7 +42,7 @@ export const GazetteerPathEditor: FC> = ({ }, [value, setGaz]); const { current, options } = useMemo(() => { - let options = [...paths]; + let options = settings?.options ? [...settings.options] : [...defaultPaths]; let current = options.find((f) => f.value === gaz?.path); if (!current && gaz) { current = { @@ -45,7 +52,7 @@ export const GazetteerPathEditor: FC> = ({ options.push(current); } return { options, current }; - }, [gaz]); + }, [gaz, settings.options]); return ( <> diff --git a/public/app/plugins/panel/geomap/types.ts b/public/app/plugins/panel/geomap/types.ts index 1f7309b178b..861be1c92cc 100644 --- a/public/app/plugins/panel/geomap/types.ts +++ b/public/app/plugins/panel/geomap/types.ts @@ -1,4 +1,4 @@ -import { MapLayerHandler, MapLayerOptions } from '@grafana/data'; +import { MapLayerHandler, MapLayerOptions, SelectableValue } from '@grafana/data'; import BaseLayer from 'ol/layer/Base'; import { Units } from 'ol/proj/Units'; import { Style } from 'ol/style'; @@ -64,6 +64,9 @@ export enum ComparisonOperation { GTE = 'gte', } +export interface GazetteerPathEditorConfigSettings { + options?: Array>; +} //------------------- // Runtime model //-------------------