From bfbc8c3c4fd059591e3d09fe1f3d14399939e3c6 Mon Sep 17 00:00:00 2001 From: Ludovic Viaud Date: Thu, 26 Jan 2023 13:10:05 +0100 Subject: [PATCH] Transforms: Add join by fields (#61322) --- .../transforms/join-by-field.json | 414 ++++++------------ .../transformers/ensureColumns.ts | 13 +- .../transformers/joinByField.test.ts | 98 ++++- .../transformers/joinByField.ts | 15 +- .../transformers/joinDataFrames.test.ts | 99 +++++ .../transformers/joinDataFrames.ts | 21 +- .../editors/JoinByFieldTransformerEditor.tsx | 86 +++- 7 files changed, 392 insertions(+), 354 deletions(-) diff --git a/devenv/dev-dashboards/transforms/join-by-field.json b/devenv/dev-dashboards/transforms/join-by-field.json index a86b3b89a89..907cee7e5d1 100644 --- a/devenv/dev-dashboards/transforms/join-by-field.json +++ b/devenv/dev-dashboards/transforms/join-by-field.json @@ -24,7 +24,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1351, "links": [], "liveNow": false, "panels": [ @@ -38,7 +37,7 @@ }, "id": 9, "panels": [], - "title": "Join by time", + "title": "Input", "type": "row" }, { @@ -49,37 +48,14 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "align": "auto", + "cellOptions": { + "type": "auto" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "inspect": false }, "mappings": [], "thresholds": { @@ -99,41 +75,41 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 1 }, "id": 11, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showHeader": true }, + "pluginVersion": "9.4.0-pre", "targets": [ { "datasource": { "type": "testdata", "uid": "PD8C576611E62080A" }, - "refId": "A", - "scenarioId": "random_walk", - "seriesCount": 4 + "rawFrameContent": "[{\r\n \"name\": \"tags\",\r\n \"fields\": [\r\n { \"name\": \"tags__time\", \"values\": [100, 101, 200] },\r\n { \"name\": \"tags__name\", \"values\": [\"v1.2\", \"v1.2b\", \"v1.3\"] }\r\n ]\r\n}]", + "refId": "tags", + "scenarioId": "raw_frame" } ], - "title": "Timeseries data", - "type": "timeseries" + "title": "tags", + "type": "table" }, { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, "fieldConfig": { "defaults": { @@ -142,7 +118,9 @@ }, "custom": { "align": "auto", - "displayMode": "auto", + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], @@ -163,13 +141,14 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 8, + "x": 8, "y": 1 }, "id": 13, "options": { "footer": { + "countRows": false, "fields": "", "reducer": [ "sum" @@ -178,24 +157,25 @@ }, "showHeader": true }, - "pluginVersion": "9.2.0-pre", + "pluginVersion": "9.4.0-pre", "targets": [ { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, - "panelId": 11, - "refId": "A" + "rawFrameContent": "[{\r\n \"name\": \"releases\",\r\n\"fields\": [\r\n { \"name\": \"releases__time\", \"values\": [150, 250] },\r\n { \"name\": \"releases__tag\", \"values\": [\"v1.2\", \"v1.3\"] }\r\n]}]", + "refId": "releases", + "scenarioId": "raw_frame" } ], - "title": "Same data (as a table)", + "title": "releases", "type": "table" }, { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, "fieldConfig": { "defaults": { @@ -204,7 +184,9 @@ }, "custom": { "align": "auto", - "displayMode": "auto", + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], @@ -224,14 +206,15 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 9 + "h": 8, + "w": 8, + "x": 16, + "y": 1 }, - "id": 16, + "id": 19, "options": { "footer": { + "countRows": false, "fields": "", "reducer": [ "sum" @@ -240,24 +223,19 @@ }, "showHeader": true }, - "pluginVersion": "9.2.0-pre", + "pluginVersion": "9.4.0-pre", "targets": [ { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, - "panelId": 11, - "refId": "A" - } - ], - "title": "OUTER join on time (default)", - "transformations": [ - { - "id": "joinByField", - "options": {} + "rawFrameContent": "[{\r\n \"name\": \"features\",\r\n\"fields\": [\r\n { \"name\": \"features__name\", \"values\": [\"A\", \"B\", \"C\", \"D\", \"E\"] },\r\n { \"name\": \"features__tag\", \"values\": [\"v1.2\", \"v1.3\", \"v1.2b\", \"v1.3\", \"v1.2\"] }\r\n]}]", + "refId": "features", + "scenarioId": "raw_frame" } ], + "title": "features", "type": "table" }, { @@ -266,11 +244,11 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 9 }, - "id": 5, + "id": 21, "panels": [], - "title": "Join by string field", + "title": "Output", "type": "row" }, { @@ -285,7 +263,9 @@ }, "custom": { "align": "auto", - "displayMode": "auto", + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], @@ -308,202 +288,61 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 10 }, - "id": 2, + "id": 23, "options": { "footer": { + "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, - "frameIndex": 0, "showHeader": true }, - "pluginVersion": "9.2.0-pre", + "pluginVersion": "9.4.0-pre", "targets": [ { - "csvContent": "OrderID,CustomerID,Time\n100,A,10000\n101,B,20000\n102,C,30000", "datasource": { "type": "testdata", "uid": "PD8C576611E62080A" }, - "refId": "Orders", - "scenarioId": "csv_content" + "rawFrameContent": "[{\r\n \"name\": \"tags\",\r\n \"fields\": [\r\n { \"name\": \"tags__time\", \"values\": [100, 101, 200] },\r\n { \"name\": \"tags__name\", \"values\": [\"v1.2\", \"v1.2b\", \"v1.3\"] }\r\n ]\r\n}]", + "refId": "tags", + "scenarioId": "raw_frame" }, { - "csvContent": "CustomerID,Name,Country\nA,Customer A,USA\nB,Customer B,Germany\nC,Customer C,Spain\nD,Customer D,Canada", "datasource": { "type": "testdata", "uid": "PD8C576611E62080A" }, - "hide": false, - "refId": "Customers", - "scenarioId": "csv_content" - } - ], - "title": "Orders", - "transformations": [], - "type": "table" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "rawFrameContent": "[{\r\n \"name\": \"releases\",\r\n \"fields\": [\r\n { \"name\": \"releases__time\", \"values\": [150, 250] },\r\n { \"name\": \"releases__tag\", \"values\": [\"v1.2\", \"v1.3\"] }\r\n]}]", + "refId": "releases", + "scenarioId": "raw_frame" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 3, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 1, - "showHeader": true - }, - "pluginVersion": "9.2.0-pre", - "targets": [ { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, - "panelId": 2, - "refId": "A" + "rawFrameContent": "[{\r\n \"name\": \"features\",\r\n \"fields\": [\r\n { \"name\": \"features__name\", \"values\": [\"A\", \"B\", \"C\", \"D\", \"E\"] },\r\n { \"name\": \"features__tag\", \"values\": [\"v1.2\", \"v1.3\", \"v1.2b\", \"v1.3\", \"v1.2\"] }\r\n]}]", + "refId": "features", + "scenarioId": "raw_frame" } ], - "title": "Customers", - "transformations": [], - "type": "table" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CustomerID" - }, - "properties": [ - { - "id": "custom.width", - "value": 101 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "OrderID" - }, - "properties": [ - { - "id": "custom.width", - "value": 89 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.2.0-pre", - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" - }, - "panelId": 2, - "refId": "A" - } - ], - "title": "OUTER join on CustomerID (keeps missing values)", + "title": "OUTER JOIN", "transformations": [ { "id": "joinByField", "options": { - "byField": "CustomerID", + "fields": { + "A": "features__name", + "features": "features__tag", + "releases": "releases__tag", + "tags": "tags__name" + }, "mode": "outer" } } @@ -512,8 +351,8 @@ }, { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, "fieldConfig": { "defaults": { @@ -522,7 +361,9 @@ }, "custom": { "align": "auto", - "displayMode": "auto", + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], @@ -539,69 +380,67 @@ ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CustomerID" - }, - "properties": [ - { - "id": "custom.width", - "value": 101 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "OrderID" - }, - "properties": [ - { - "id": "custom.width", - "value": 89 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 10 }, - "id": 7, + "id": 24, "options": { "footer": { + "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [] + "showHeader": true }, - "pluginVersion": "9.2.0-pre", + "pluginVersion": "9.4.0-pre", "targets": [ { "datasource": { - "type": "datasource", - "uid": "-- Dashboard --" + "type": "testdata", + "uid": "PD8C576611E62080A" }, - "panelId": 2, - "refId": "A" + "rawFrameContent": "[{\r\n \"name\": \"tags\",\r\n \"fields\": [\r\n { \"name\": \"tags__time\", \"values\": [100, 101, 200] },\r\n { \"name\": \"tags__name\", \"values\": [\"v1.2\", \"v1.2b\", \"v1.3\"] }\r\n ]\r\n}]", + "refId": "tags", + "scenarioId": "raw_frame" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "rawFrameContent": "[{\r\n \"name\": \"releases\",\r\n \"fields\": [\r\n { \"name\": \"releases__time\", \"values\": [150, 250] },\r\n { \"name\": \"releases__tag\", \"values\": [\"v1.2\", \"v1.3\"] }\r\n]}]", + "refId": "releases", + "scenarioId": "raw_frame" + }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "rawFrameContent": "[{\r\n \"name\": \"features\",\r\n \"fields\": [\r\n { \"name\": \"features__name\", \"values\": [\"A\", \"B\", \"C\", \"D\", \"E\"] },\r\n { \"name\": \"features__tag\", \"values\": [\"v1.2\", \"v1.3\", \"v1.2b\", \"v1.3\", \"v1.2\"] }\r\n]}]", + "refId": "features", + "scenarioId": "raw_frame" } ], - "title": "INNER join on CustomerID ", + "title": "INNER JOIN", "transformations": [ { "id": "joinByField", "options": { - "byField": "CustomerID", + "fields": { + "A": "features__name", + "features": "features__tag", + "releases": "releases__tag", + "tags": "tags__name" + }, "mode": "inner" } } @@ -609,7 +448,8 @@ "type": "table" } ], - "schemaVersion": 37, + "revision": 1, + "schemaVersion": 38, "style": "dark", "tags": [ "gdev", @@ -626,6 +466,6 @@ "timezone": "", "title": "Join by field", "uid": "gw0K4rmVz", - "version": 6, + "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts index 4968336c236..9cca6fead37 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.ts @@ -20,12 +20,13 @@ export const ensureColumnsTransformer: SynchronousDataTransformerInfo = { const timeFieldName = findConsistentTimeFieldName(frames); if (frames.length > 1 && timeFieldName) { - return joinByFieldTransformer.transformer( - { - byField: timeFieldName, - }, - ctx - )(frames); + const fields: { [key: string]: string } = {}; + for (const frame of frames) { + if (frame.refId) { + fields[frame.refId] = timeFieldName; + } + } + return joinByFieldTransformer.transformer({ fields }, ctx)(frames); } return frames; }, diff --git a/packages/grafana-data/src/transformations/transformers/joinByField.test.ts b/packages/grafana-data/src/transformations/transformers/joinByField.test.ts index 1a1026017b3..5326e86da46 100644 --- a/packages/grafana-data/src/transformations/transformers/joinByField.test.ts +++ b/packages/grafana-data/src/transformations/transformers/joinByField.test.ts @@ -15,6 +15,7 @@ describe('JOIN Transformer', () => { describe('outer join', () => { const everySecondSeries = toDataFrame({ name: 'even', + refId: 'even', fields: [ { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] }, @@ -24,6 +25,7 @@ describe('JOIN Transformer', () => { const everyOtherSecondSeries = toDataFrame({ name: 'odd', + refId: 'odd', fields: [ { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] }, { name: 'temperature', type: FieldType.number, values: [11.1, 11.3, 11.5, 11.7] }, @@ -33,9 +35,12 @@ describe('JOIN Transformer', () => { it('joins by time field', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + even: 'time', + odd: 'time', + }, }, }; @@ -135,9 +140,12 @@ describe('JOIN Transformer', () => { it('joins by temperature field', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'temperature', + fields: { + even: 'temperature', + odd: 'temperature', + }, }, }; @@ -145,6 +153,7 @@ describe('JOIN Transformer', () => { (received) => { const data = received[0]; const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` [ { @@ -251,9 +260,12 @@ describe('JOIN Transformer', () => { it('joins by time field in reverse order', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + even: 'time', + odd: 'time', + }, }, }; @@ -265,6 +277,7 @@ describe('JOIN Transformer', () => { (received) => { const data = received[0]; const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` [ { @@ -376,9 +389,12 @@ describe('JOIN Transformer', () => { it('when dataframe and field share the same name then use the field name', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + even: 'time', + odd: 'time', + }, }, }; @@ -439,9 +455,12 @@ describe('JOIN Transformer', () => { it('joins if fields are missing', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + even: 'time', + odd: 'time', + }, }, }; @@ -517,9 +536,12 @@ describe('JOIN Transformer', () => { it('handles duplicate field name', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + even: 'time', + odd: 'time', + }, }, }; @@ -580,6 +602,7 @@ describe('JOIN Transformer', () => { describe('inner join', () => { const seriesA = toDataFrame({ name: 'A', + refId: 'A', fields: [ { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] }, @@ -589,6 +612,7 @@ describe('JOIN Transformer', () => { const seriesB = toDataFrame({ name: 'B', + refId: 'B', fields: [ { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] }, { name: 'temperature', type: FieldType.number, values: [11.1, 10.3, 10.5, 11.7] }, @@ -598,9 +622,12 @@ describe('JOIN Transformer', () => { it('inner joins by time field', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + A: 'time', + B: 'time', + }, mode: JoinMode.inner, }, }; @@ -679,9 +706,12 @@ describe('JOIN Transformer', () => { it('inner joins by temperature field', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'temperature', + fields: { + A: 'temperature', + B: 'temperature', + }, mode: JoinMode.inner, }, }; @@ -764,9 +794,12 @@ describe('JOIN Transformer', () => { it('inner joins by time field in reverse order', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + A: 'time', + B: 'time', + }, mode: JoinMode.inner, }, }; @@ -852,6 +885,7 @@ describe('JOIN Transformer', () => { describe('Field names', () => { const seriesWithSameFieldAndDataFrameName = toDataFrame({ name: 'temperature', + refId: 'temperature', fields: [ { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] }, @@ -860,6 +894,7 @@ describe('JOIN Transformer', () => { const seriesB = toDataFrame({ name: 'B', + refId: 'B', fields: [ { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, { name: 'temperature', type: FieldType.number, values: [2, 4, 6, 8] }, @@ -868,9 +903,12 @@ describe('JOIN Transformer', () => { it('when dataframe and field share the same name then use the field name', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + temperature: 'time', + B: 'time', + }, mode: JoinMode.inner, }, }; @@ -932,15 +970,20 @@ describe('JOIN Transformer', () => { it('joins if fields are missing', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + A: 'time', + B: 'time', + C: 'time', + }, mode: JoinMode.inner, }, }; const frame1 = toDataFrame({ name: 'A', + refId: 'A', fields: [ { name: 'time', type: FieldType.time, values: [1, 2, 3] }, { name: 'temperature', type: FieldType.number, values: [10, 11, 12] }, @@ -949,11 +992,13 @@ describe('JOIN Transformer', () => { const frame2 = toDataFrame({ name: 'B', + refId: 'B', fields: [], }); const frame3 = toDataFrame({ name: 'C', + refId: 'C', fields: [ { name: 'time', type: FieldType.time, values: [1, 2, 3] }, { name: 'temperature', type: FieldType.number, values: [20, 22, 24] }, @@ -1011,14 +1056,18 @@ describe('JOIN Transformer', () => { it('handles duplicate field name', async () => { const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, + id: DataTransformerID.joinByField, options: { - byField: 'time', + fields: { + frame1: 'time', + frame2: 'time', + }, mode: JoinMode.inner, }, }; const frame1 = toDataFrame({ + refId: 'frame1', fields: [ { name: 'time', type: FieldType.time, values: [1] }, { name: 'temperature', type: FieldType.number, values: [10] }, @@ -1026,6 +1075,7 @@ describe('JOIN Transformer', () => { }); const frame2 = toDataFrame({ + refId: 'frame2', fields: [ { name: 'time', type: FieldType.time, values: [1] }, { name: 'temperature', type: FieldType.number, values: [20] }, diff --git a/packages/grafana-data/src/transformations/transformers/joinByField.ts b/packages/grafana-data/src/transformations/transformers/joinByField.ts index e6c1d386fa6..2946dd4125b 100644 --- a/packages/grafana-data/src/transformations/transformers/joinByField.ts +++ b/packages/grafana-data/src/transformations/transformers/joinByField.ts @@ -1,8 +1,6 @@ import { map } from 'rxjs/operators'; -import { DataFrame, SynchronousDataTransformerInfo, FieldMatcher } from '../../types'; -import { fieldMatchers } from '../matchers'; -import { FieldMatcherID } from '../matchers/ids'; +import { DataFrame, SynchronousDataTransformerInfo } from '../../types'; import { DataTransformerID } from './ids'; import { joinDataFrames } from './joinDataFrames'; @@ -13,7 +11,7 @@ export enum JoinMode { } export interface JoinByFieldOptions { - byField?: string; // empty will pick the field automatically + fields?: { [key: string]: string }; // empty will pick the field automatically mode?: JoinMode; } @@ -24,7 +22,7 @@ export const joinByFieldTransformer: SynchronousDataTransformerInfo joinByFieldTransformer.transformer(options, ctx)(data))), transformer: (options: JoinByFieldOptions) => { - let joinBy: FieldMatcher | undefined = undefined; return (data: DataFrame[]) => { if (data.length > 1) { - if (options.byField && !joinBy) { - joinBy = fieldMatchers.get(FieldMatcherID.byName).get(options.byField); - } - const joined = joinDataFrames({ frames: data, joinBy, mode: options.mode }); + const joined = joinDataFrames({ frames: data, mode: options.mode, fields: options.fields }); if (joined) { return [joined]; } } + return data; }; }, diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts index 5f4ef23427b..83dd9469f6b 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts @@ -359,4 +359,103 @@ describe('align frames', () => { expect(isLikelyAscendingVector(new ArrayVector([null, 1, null]), 3)).toBeTruthy(); }); }); + + describe('should perform a join on custom fields', () => { + const tags = toDataFrame({ + refId: 'tags', + fields: [ + { name: 'tags__time', type: FieldType.time, values: [100, 101, 200] }, + { name: 'tags__name', type: FieldType.string, values: ['v1.2', 'v1.2b', 'v1.3'] }, + ], + }); + + const releases = toDataFrame({ + refId: 'releases', + fields: [ + { name: 'releases__time', type: FieldType.time, values: [150, 250] }, + { name: 'releases__tag', type: FieldType.string, values: ['v1.2', 'v1.3'] }, + ], + }); + + const features = toDataFrame({ + refId: 'features', + fields: [ + { name: 'features__name', type: FieldType.string, values: ['A', 'B', 'C', 'D', 'E'] }, + { name: 'features__tag', type: FieldType.time, values: ['v1.2', 'v1.3', 'v1.2b', 'v1.3', 'v1.2'] }, + ], + }); + + it('should perform an outer join', () => { + const out = joinDataFrames({ + frames: [tags, releases, features], + fields: { + tags: 'tags__name', + releases: 'releases__tag', + features: 'features__tag', + }, + })!; + + expect( + out.fields.map((f) => ({ + name: f.name, + values: f.values.toArray(), + })) + ).toEqual([ + { + name: 'tags__name', + values: ['v1.2', 'v1.2b', 'v1.3'], + }, + { + name: 'tags__time', + values: [100, 101, 200], + }, + { + name: 'releases__time', + values: [150, undefined, 250], + }, + { + name: 'features__name', + values: ['E', 'C', 'D'], + }, + ]); + }); + + it('should perform an inner join', () => { + const out = joinDataFrames({ + frames: [tags, releases, features], + fields: { + tags: 'tags__name', + releases: 'releases__tag', + features: 'features__tag', + }, + mode: JoinMode.inner, + })!; + + const mappedOut = out.fields.map((f) => ({ + name: f.name, + values: f.values.toArray(), + })); + + const expected = [ + { + name: 'tags__name', + values: ['v1.2', 'v1.3'], + }, + { + name: 'tags__time', + values: [100, 200], + }, + { + name: 'releases__time', + values: [150, 250], + }, + { + name: 'features__name', + values: ['E', 'D'], + }, + ]; + + expect(JSON.stringify(mappedOut)).toEqual(JSON.stringify(expected)); + }); + }); }); diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts index 767161d2f9a..dd09c373bfb 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts @@ -47,6 +47,11 @@ export interface JoinOptions { */ joinBy?: FieldMatcher; + /** + * The fields to join on + */ + fields?: { [key: string]: string }; + /** * Optionally filter the non-join fields */ @@ -63,8 +68,16 @@ export interface JoinOptions { mode?: JoinMode; } -function getJoinMatcher(options: JoinOptions): FieldMatcher { - return options.joinBy ?? pickBestJoinField(options.frames); +function getJoinMatcher(options: JoinOptions, refId: string | undefined): FieldMatcher { + if (options.joinBy) { + return options.joinBy; + } + + if (!options.fields || !refId) { + return pickBestJoinField(options.frames); + } + + return fieldMatchers.get(FieldMatcherID.byName).get(options.fields[refId]); } /** @@ -95,7 +108,7 @@ export function joinDataFrames(options: JoinOptions): DataFrame | undefined { let frame = options.frames[0]; let frameCopy = frame; - const joinFieldMatcher = getJoinMatcher(options); + const joinFieldMatcher = getJoinMatcher(options, frame.refId); let joinIndex = frameCopy.fields.findIndex((f) => joinFieldMatcher(f, frameCopy, options.frames)); if (options.keepOriginIndices) { @@ -152,10 +165,10 @@ export function joinDataFrames(options: JoinOptions): DataFrame | undefined { const nullModes: JoinNullMode[][] = []; const allData: AlignedData[] = []; const originalFields: Field[] = []; - const joinFieldMatcher = getJoinMatcher(options); for (let frameIndex = 0; frameIndex < options.frames.length; frameIndex++) { const frame = options.frames[frameIndex]; + const joinFieldMatcher = getJoinMatcher(options, frame.refId); if (!frame || !frame.fields?.length) { continue; // skip the frame diff --git a/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx b/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx index cffa84015e0..1b0977455d6 100644 --- a/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx +++ b/public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx @@ -1,4 +1,4 @@ -import React, { useCallback } from 'react'; +import React, { useCallback, useEffect } from 'react'; import { DataTransformerID, @@ -6,11 +6,10 @@ import { standardTransformers, TransformerRegistryItem, TransformerUIProps, + DataFrame, } from '@grafana/data'; import { JoinByFieldOptions, JoinMode } from '@grafana/data/src/transformations/transformers/joinByField'; -import { Select, InlineFieldRow, InlineField } from '@grafana/ui'; - -import { useAllFieldNamesFromDataFrames } from '../utils'; +import { Select, InlineFieldRow, InlineField, Checkbox, HorizontalGroup } from '@grafana/ui'; const modes = [ { value: JoinMode.outer, label: 'OUTER', description: 'Keep all rows from any table with a value' }, @@ -18,14 +17,44 @@ const modes = [ ]; export function SeriesToFieldsTransformerEditor({ input, options, onChange }: TransformerUIProps) { - const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item })); + useEffect(() => { + if (options.fields && !Object.keys(options.fields).length && input.length && input[0].refId) { + options.fields[input[0].refId] = input[0].fields[0].name; + onChange({ ...options }); + } + }, [onChange, options, input]); + + const onToggleDataFrame = useCallback( + (dataFrame: DataFrame) => { + if (!dataFrame.refId) { + return; + } + + if (options.fields) { + if (dataFrame.refId in options.fields) { + if (Object.keys(options.fields).length === 1) { + return; + } + + delete options.fields[dataFrame.refId]; + } else { + options.fields[dataFrame.refId] = dataFrame.fields[0].name; + } + } + + onChange({ ...options }); + }, + [onChange, options] + ); const onSelectField = useCallback( - (value: SelectableValue) => { - onChange({ - ...options, - byField: value?.value, - }); + (queryRefId: string | undefined, fieldName: SelectableValue) => { + if (queryRefId && fieldName.value) { + onChange({ + ...options, + fields: { ...options.fields, [queryRefId]: fieldName.value }, + }); + } }, [onChange, options] ); @@ -34,7 +63,7 @@ export function SeriesToFieldsTransformerEditor({ input, options, onChange }: Tr (value: SelectableValue) => { onChange({ ...options, - mode: value?.value, + mode: value?.value || JoinMode.outer, }); }, [onChange, options] @@ -44,20 +73,31 @@ export function SeriesToFieldsTransformerEditor({ input, options, onChange }: Tr <> - + ({ label: field.name, value: field.name }))} + value={dataFrame.refId ? (options.fields || {})[dataFrame.refId] : dataFrame.fields[0].name} + onChange={(fieldName) => onSelectField(dataFrame.refId, fieldName)} + /> + + + + ))} ); }