From 47f3073ab84f8654bfda65edb4112cd73d065ce4 Mon Sep 17 00:00:00 2001 From: Leon Sorokin Date: Fri, 13 Jun 2025 13:47:30 -0500 Subject: [PATCH] Transformations: Speed up INNER JOIN by ~200x, OUTER by ~20x (#105592) --- .../transformers/joinDataFrames.test.ts | 18 +- .../transformers/joinDataFrames.ts | 338 ++++++++++-------- 2 files changed, 195 insertions(+), 161 deletions(-) diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts index 0515131be30..956dc272604 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts @@ -142,20 +142,20 @@ describe('align frames', () => { { name: 'gender', type: FieldType.string, - values: ['NON-BINARY', 'MALE', 'MALE', 'FEMALE', 'FEMALE', 'NON-BINARY'], + values: ['NON-BINARY', 'MALE', 'MALE', 'FEMALE', 'FEMALE', 'NON-BINARY', 'COW'], }, { name: 'day', type: FieldType.string, - values: ['Wednesday', 'Tuesday', 'Monday', 'Wednesday', 'Tuesday', 'Monday'], + values: ['Wednesday', 'Tuesday', 'Monday', 'Wednesday', 'Tuesday', 'Monday', 'Monday'], }, - { name: 'count', type: FieldType.number, values: [18, 72, 13, 17, 71, 7] }, + { name: 'count', type: FieldType.number, values: [18, 72, 13, 17, 71, 7, 1] }, ], }); const tableData2 = toDataFrame({ fields: [ - { name: 'gender', type: FieldType.string, values: ['MALE', 'NON-BINARY', 'FEMALE'] }, - { name: 'count', type: FieldType.number, values: [103, 95, 201] }, + { name: 'gender', type: FieldType.string, values: ['MALE', 'NON-BINARY', 'FEMALE', 'DOG'] }, + { name: 'count', type: FieldType.number, values: [103, 95, 201, 6] }, ], }); @@ -181,6 +181,8 @@ describe('align frames', () => { "FEMALE", "FEMALE", "NON-BINARY", + "COW", + "DOG", ], }, { @@ -192,6 +194,8 @@ describe('align frames', () => { "Wednesday", "Tuesday", "Monday", + "Monday", + null, ], }, { @@ -203,6 +207,8 @@ describe('align frames', () => { 17, 71, 7, + 1, + null, ], }, { @@ -214,6 +220,8 @@ describe('align frames', () => { 201, 201, 95, + null, + 6, ], }, ] diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts index 98f28ff9fce..ee621286516 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts @@ -167,10 +167,6 @@ export function joinDataFrames(options: JoinOptions): DataFrame | undefined { const nullModes: JoinNullMode[][] = []; const allData: AlignedData[] = []; const originalFields: Field[] = []; - // store frame field order for tabular data join - const originalFieldsOrderByFrame: number[][] = []; - // all other fields that are not the join on are in the 1+ position (join is always the 0) - let fieldsOrder = 1; const joinFieldMatcher = getJoinMatcher(options); for (let frameIndex = 0; frameIndex < options.frames.length; frameIndex++) { @@ -183,7 +179,6 @@ export function joinDataFrames(options: JoinOptions): DataFrame | undefined { const nullModesFrame: JoinNullMode[] = [NULL_REMOVE]; let join: Field | undefined = undefined; let fields: Field[] = []; - let frameFieldsOrder = []; for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { const field = frame.fields[fieldIndex]; @@ -243,21 +238,16 @@ export function joinDataFrames(options: JoinOptions): DataFrame | undefined { // clear field displayName state delete field.state?.displayName; } - // store frame field order for tabular data join - frameFieldsOrder.push(fieldsOrder); - fieldsOrder++; } - // store frame field order for tabular data join - originalFieldsOrderByFrame.push(frameFieldsOrder); allData.push(a); } let joined: Array> = []; if (options.mode === JoinMode.outerTabular) { - joined = joinOuterTabular(allData, originalFieldsOrderByFrame, originalFields.length, nullModes); + joined = joinTabular(allData, true); } else if (options.mode === JoinMode.inner) { - joined = joinInner(allData); + joined = joinTabular(allData); } else { joined = join(allData, nullModes, options.mode); } @@ -272,165 +262,201 @@ export function joinDataFrames(options: JoinOptions): DataFrame | undefined { }; } -// The following full outer join allows for multiple/duplicated joined fields values where as the performant join from uplot creates a unique set of field values to be joined on -// http://www.silota.com/docs/recipes/sql-join-tutorial-javascript-examples.html -// The frame field value which is used join on is sorted to the 0 position of each table data in both tables and nullModes -// (not sure if we need nullModes) for nullModes, the field to join on is given NULL_REMOVE and all other fields are given NULL_EXPAND -function joinOuterTabular( - tables: AlignedData[], - originalFieldsOrderByFrame: number[][], - numberOfFields: number, - nullModes?: number[][] -) { - // we will iterate through all frames and check frames for matches preventing duplicates. - // we will store each matched frame "row" or field values at the same index in the following hash. - let duplicateHash: { [key: string]: Array } = {}; - - // iterate through the tables (frames) - // for each frame we get the field data where the data in the 0 pos is the value to join on - for (let tableIdx = 0; tableIdx < tables.length; tableIdx++) { - // the table (frame) to check for matches in other tables - let table = tables[tableIdx]; - // the field value to join on (the join value is always in the 0 position) - let joinOnTableField = table[0]; - - // now we iterate through the other table (frame) data to look for matches - for (let otherTablesIdx = 0; otherTablesIdx < tables.length; otherTablesIdx++) { - // do not match on the same table - if (otherTablesIdx === tableIdx) { - continue; - } - - let otherTable = tables[otherTablesIdx]; - let otherTableJoinOnField = otherTable[0]; - - // iterate through the field to join on from the first table - for ( - let joinTableFieldValuesIdx = 0; - joinTableFieldValuesIdx < joinOnTableField.length; - joinTableFieldValuesIdx++ - ) { - // create the joined data - // this has the orignalFields length and should start out undefined - // joined row + number of other fields in each frame - // the order of each field is important in how we - // 1 check for duplicates - // 2 transform the row back into fields for the joined frame - // 3 when there is no match for the row we keep the vals undefined - const tableJoinOnValue = joinOnTableField[joinTableFieldValuesIdx]; - const allOtherFields = numberOfFields - 1; - let joinedRow: Array = [tableJoinOnValue].concat(new Array(allOtherFields)); - - let tableFieldValIdx = 0; - for (let fieldsIdx = 1; fieldsIdx < table.length; fieldsIdx++) { - const joinRowIdx = originalFieldsOrderByFrame[tableIdx][tableFieldValIdx]; - joinedRow[joinRowIdx] = table[fieldsIdx][joinTableFieldValuesIdx]; - tableFieldValIdx++; - } - - for (let otherTableValuesIdx = 0; otherTableValuesIdx < otherTableJoinOnField.length; otherTableValuesIdx++) { - if (joinOnTableField[joinTableFieldValuesIdx] === otherTableJoinOnField[otherTableValuesIdx]) { - let tableFieldValIdx = 0; - for (let fieldsIdx = 1; fieldsIdx < otherTable.length; fieldsIdx++) { - const joinRowIdx = originalFieldsOrderByFrame[otherTablesIdx][tableFieldValIdx]; - joinedRow[joinRowIdx] = otherTable[fieldsIdx][otherTableValuesIdx]; - tableFieldValIdx++; - } - - break; - } - } - - // prevent duplicates by entering rows in a hash where keys are the rows - duplicateHash[JSON.stringify(joinedRow)] = joinedRow; - } - } - } - - // transform the joined rows into data for a dataframe - let data: Array> = []; - for (let field = 0; field < numberOfFields; field++) { - data.push(new Array(0)); - } - - for (let key in duplicateHash) { - const row = duplicateHash[key]; - - for (let valIdx = 0; valIdx < row.length; valIdx++) { - data[valIdx].push(row[valIdx]); - } - } - - return data; -} - /** - * This function performs a sql-style inner join on tabular data; - * it will combine records from two tables whenever there are matching - * values in a field common to both tables. - * - * NOTE: This function implicitly assumes that the first array in each AlignedData - * contains the values to join on. It doesn't explicitly specify a column field to join on, - * but rather uses the 0th position of the arrays (AlignedData[0]) to determine the joining keys. - * Then, when processing the tables, the function iterates over the values in the `xValues` - * (the joining keys) array and checks if the current row `currentRow` already includes the value. - * If a matching value is found, it joins the corresponding values from the remaining arrays `yValues` - * (all other non-joining key arrays) to create a new row in the joined table. - * - * @param {AlignedData[]} tables - The tables to join. - * - * @returns {Array>} The joined tables as an array of arrays, where each array represents a row in the joined table. + * SQL-style join of tables, using the first column in each */ -function joinInner(tables: AlignedData[]): Array> { - const joinedTables: Array> = []; +function joinTabular(tables: AlignedData[], outer = false) { + // console.time('joinTabular'); - // Recursive function to perform the inner join. - const joinTables = ( - currentTables: AlignedData[], - currentIndex: number, - currentRow: Array - ) => { - if (currentIndex === currentTables.length) { - // Base case: all tables have been joined, add the current row to the final result. - joinedTables.push(currentRow); - return; + let ltable = tables[0]; + let lfield = ltable[0]; + + // iterate tables, merging right table with left, with the result becoming the new left + // rinse and repeat for each tables in the array + for (let ti = 1; ti < tables.length; ti++) { + let rtable = tables[ti]; + let rfield = rtable[0]; + + /** + * Build an inverted index of the right table's join column like { "foo": [1,2,3], "bar": [7,12], ... } + * where the keys are unique values and the arrays are indices where these values were found + */ + // console.time('index right'); + let index: Record = {}; + + for (let i = 0; i < rfield.length; i++) { + let val = rfield[i]; + + let idxs = index[val]; + + if (idxs == null) { + idxs = index[val] = []; + } + + idxs.push(i); } + // console.timeEnd('index right'); - const currentTable = currentTables[currentIndex]; - const [xValues, ...yValues] = currentTable; + /** + * Loop over the left table's join column and match each non-null value to the right index, + * copying the matched ridxs array into new matched list, like [33, [45,79,233]], where first + * value is left idx and second value is right idxs + * + * Also keep track of unmatched or null left values for outer join, since we'll need to include these + */ + let matchedKeys = new Set(); + let unmatchedLeft = []; + let unmatchedRight = []; - for (let i = 0; i < xValues.length; i++) { - const value = xValues[i]; + // console.time('match left'); + let matched: Array<[lidx: number, ridxs: number[]]> = []; - if (currentIndex === 0 || currentRow.includes(value)) { - const newRow = [...currentRow]; + // count of total number of output rows, so we can + // pre-allocate the final array size during materialization + let count = 0; - if (currentIndex === 0) { - newRow.push(value); + for (let i = 0; i < lfield.length; i++) { + let v = lfield[i]; + + if (v != null) { + let idxs = index[v]; + + if (idxs != null) { + matched.push([i, idxs]); + count += idxs.length; + outer && matchedKeys.add(v); + } else if (outer) { + unmatchedLeft.push(i); } - - for (let j = 0; j < yValues.length; j++) { - newRow.push(yValues[j][i]); - } - - // Recursive call for the next table - joinTables(currentTables, currentIndex + 1, newRow); + } else if (outer) { + unmatchedLeft.push(i); } } - }; + count += unmatchedLeft.length; + // console.timeEnd('match left'); - // Start the recursive join process. - joinTables(tables, 0, []); + /** + * For outer joins, also loop over the right index to record unmatched values + */ + // console.time('unmatched right'); + if (outer) { + for (let k in index) { + if (!matchedKeys.has(k)) { + unmatchedRight.push(...index[k]); + } + } + count += unmatchedRight.length; + } + // console.timeEnd('unmatched right'); - // Check if joinedTables is empty before transposing. No need to transpose if there are no joined tables. - if (joinedTables.length === 0) { - const fieldCount = tables.reduce((count, table) => count + (table.length - 1), 1); - return Array.from({ length: fieldCount }, () => []); + /** + * Now we can use matched, unmatchedLeft, unmatchedRight, ltable, and rtable to assemble the final table + * Instead of using 3-deep nested loops, we eliminate the loops over the known column structure + * For this we compile a new function using the schemas from both tables, and filling that struct by looping + * over the matched lookup array, then appending the unmatched left rows (and null-filling the right values), + * then appending the unmatched right rows (and null-filling the left values). + * + * The assembled function looks something like this when joining 2-col left + 2-col right: + * + * function anonymous(matched, unmatchedLeft, unmatchedRight, ltable, rtable) { + * const joined = [Array(99991),Array(99991),Array(99991)]; + * + * let rowIdx = 0; + * + * for (let i = 0; i < matched.length; i++) { + * let [lidx, ridxs] = matched[i]; + * + * for (let j = 0; j < ridxs.length; j++, rowIdx++) { + * let ridx = ridxs[j]; + * joined[0][rowIdx] = ltable[0][lidx]; + * joined[1][rowIdx] = ltable[1][lidx]; + * joined[2][rowIdx] = rtable[1][ridx]; + * } + * } + * + * for (let i = 0; i < unmatchedLeft.length; i++, rowIdx++) { + * let lidx = unmatchedLeft[i]; + * joined[0][rowIdx] = ltable[0][lidx]; + * joined[1][rowIdx] = ltable[1][lidx]; + * joined[2][rowIdx] = null; + * } + * + * for (let i = 0; i < unmatchedRight.length; i++, rowIdx++) { + * let ridx = unmatchedRight[i]; + * joined[0][rowIdx] = rtable[0][ridx]; + * joined[1][rowIdx] = null; + * joined[2][rowIdx] = rtable[1][ridx]; + * } + * + * return joined; + * } + */ + // console.time('materialize'); + let outFieldsTpl = Array.from({ length: ltable.length + rtable.length - 1 }, () => `Array(${count})`).join(','); + let copyLeftRowTpl = ltable.map((c, i) => `joined[${i}][rowIdx] = ltable[${i}][lidx]`).join(';'); + // (skips join field in right table) + let copyRightRowTpl = rtable + .slice(1) + .map((c, i) => `joined[${ltable.length + i}][rowIdx] = rtable[${i + 1}][ridx]`) + .join(';'); + + // for outer joins, when we null-fill the left row values, we still populate the first (join) column + // with the right row's join column value, rather than omitting it as we do for matched left/right where + // that value is already filled by the left row + let nullLeftRowTpl = ltable + .map((c, i) => `joined[${i}][rowIdx] = ${i === 0 ? `rtable[${i}][ridx]` : `null`}`) + .join(';'); + // (skips join field in right table) + let nullRightRowTpl = rtable.slice(1).map((c, i) => `joined[${ltable.length + i}][rowIdx] = null`); + + let materialize = new Function( + 'matched', + 'unmatchedLeft', + 'unmatchedRight', + 'ltable', + 'rtable', + ` + const joined = [${outFieldsTpl}]; + + let rowIdx = 0; + + for (let i = 0; i < matched.length; i++) { + let [lidx, ridxs] = matched[i]; + + for (let j = 0; j < ridxs.length; j++, rowIdx++) { + let ridx = ridxs[j]; + ${copyLeftRowTpl}; + ${copyRightRowTpl}; + } + } + + for (let i = 0; i < unmatchedLeft.length; i++, rowIdx++) { + let lidx = unmatchedLeft[i]; + ${copyLeftRowTpl}; + ${nullRightRowTpl}; + } + + for (let i = 0; i < unmatchedRight.length; i++, rowIdx++) { + let ridx = unmatchedRight[i]; + ${nullLeftRowTpl}; + ${copyRightRowTpl}; + } + + return joined; + ` + ); + + let joined = materialize(matched, unmatchedLeft, unmatchedRight, ltable, rtable); + // console.timeEnd('materialize'); + + ltable = joined; + lfield = ltable[0]; } - // Transpose the joined tables to get the desired output format. - // This essentially flips the rows and columns back to the stucture of the original `tables`. - return joinedTables[0].map((_, colIndex) => joinedTables.map((row) => row[colIndex])); + // console.timeEnd('joinTabular'); + + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return ltable as Array>; } //--------------------------------------------------------------------------------