From ed69a1f16d3ee042a6ef056238aedaa08c1d7f7e Mon Sep 17 00:00:00 2001 From: Oscar Kilhed Date: Mon, 8 Sep 2025 16:23:06 +0200 Subject: [PATCH] Dashboards: Fix UTF-8 characters not working with excel downloads by replacing download for excel with excel compatibility mode. (#110099) * convert to utf-16le * Fix tests, remove download for excel * update i18n * Update copy * update test --- .../features/inspector/InspectDataOptions.tsx | 24 +++++++------- .../app/features/inspector/InspectDataTab.tsx | 16 ++++----- .../features/inspector/utils/download.test.ts | 33 ++++++++++++------- .../app/features/inspector/utils/download.ts | 30 +++++++++++++---- public/locales/en-US/grafana.json | 5 ++- 5 files changed, 68 insertions(+), 40 deletions(-) diff --git a/public/app/features/inspector/InspectDataOptions.tsx b/public/app/features/inspector/InspectDataOptions.tsx index cc37517a178..1dc2a651f10 100644 --- a/public/app/features/inspector/InspectDataOptions.tsx +++ b/public/app/features/inspector/InspectDataOptions.tsx @@ -14,14 +14,14 @@ interface Props { dataFrames: DataFrame[]; transformationOptions: Array>; selectedDataFrame: number | DataTransformerID; - downloadForExcel: boolean; onDataFrameChange: (item: SelectableValue) => void; - toggleDownloadForExcel: () => void; data?: DataFrame[]; hasTransformations?: boolean; formattedDataDescription?: string; onOptionsChange?: (options: GetDataOptions) => void; actions?: React.ReactNode; + excelCompatibilityMode: boolean; + toggleExcelCompatibilityMode: () => void; } export const InspectDataOptions = ({ @@ -35,8 +35,8 @@ export const InspectDataOptions = ({ transformationOptions, selectedDataFrame, onDataFrameChange, - downloadForExcel, - toggleDownloadForExcel, + excelCompatibilityMode, + toggleExcelCompatibilityMode, }: Props) => { const styles = useStyles2(getPanelInspectorStyles2); @@ -82,10 +82,6 @@ export const InspectDataOptions = ({ } } - if (downloadForExcel) { - parts.push(t('dashboard.inspect-data.excel-header', 'Excel header')); - } - return parts.join(', '); } @@ -147,13 +143,17 @@ export const InspectDataOptions = ({ )} - + diff --git a/public/app/features/inspector/InspectDataTab.tsx b/public/app/features/inspector/InspectDataTab.tsx index a4e7ceb01ed..8aba53622cc 100644 --- a/public/app/features/inspector/InspectDataTab.tsx +++ b/public/app/features/inspector/InspectDataTab.tsx @@ -48,7 +48,7 @@ interface State { dataFrameIndex: number; transformationOptions: Array>; transformedData: DataFrame[]; - downloadForExcel: boolean; + excelCompatibilityMode: boolean; } export class InspectDataTab extends PureComponent { @@ -61,7 +61,7 @@ export class InspectDataTab extends PureComponent { transformId: DataTransformerID.noop, transformationOptions: buildTransformationOptions(), transformedData: props.data ?? [], - downloadForExcel: false, + excelCompatibilityMode: false, }; } @@ -102,7 +102,7 @@ export class InspectDataTab extends PureComponent { reportInteraction('grafana_logs_download_clicked', { app: this.props.app, format: 'csv' }); } - downloadDataFrameAsCsv(dataFrame, dataName, { useExcelHeader: this.state.downloadForExcel }, transformId); + downloadDataFrameAsCsv(dataFrame, dataName, {}, transformId, this.state.excelCompatibilityMode); } onExportLogsAsTxt = () => { @@ -167,9 +167,9 @@ export class InspectDataTab extends PureComponent { }); }; - onToggleDownloadForExcel = () => { + onToggleExcelCompatibilityMode = () => { this.setState((prevState) => ({ - downloadForExcel: !prevState.downloadForExcel, + excelCompatibilityMode: !prevState.excelCompatibilityMode, })); }; @@ -244,7 +244,7 @@ export class InspectDataTab extends PureComponent { render() { const { isLoading, options, data, formattedDataDescription, onOptionsChange, hasTransformations } = this.props; - const { dataFrameIndex, transformationOptions, selectedDataFrame, downloadForExcel } = this.state; + const { dataFrameIndex, transformationOptions, selectedDataFrame, excelCompatibilityMode } = this.state; const styles = getPanelInspectorStyles(); if (isLoading) { @@ -282,11 +282,11 @@ export class InspectDataTab extends PureComponent { dataFrames={dataFrames} transformationOptions={transformationOptions} selectedDataFrame={selectedDataFrame} - downloadForExcel={downloadForExcel} formattedDataDescription={formattedDataDescription} onOptionsChange={onOptionsChange} onDataFrameChange={this.onDataFrameChange} - toggleDownloadForExcel={this.onToggleDownloadForExcel} + excelCompatibilityMode={excelCompatibilityMode} + toggleExcelCompatibilityMode={this.onToggleExcelCompatibilityMode} actions={this.renderActions(dataFrames, hasLogs, hasTraces, hasServiceGraph)} /> diff --git a/public/app/features/inspector/utils/download.test.ts b/public/app/features/inspector/utils/download.test.ts index 9b98d629307..b2e76a29c89 100644 --- a/public/app/features/inspector/utils/download.test.ts +++ b/public/app/features/inspector/utils/download.test.ts @@ -25,11 +25,11 @@ describe('inspector download', () => { ], }, data: { - values: [[100], ['a'], [1]], + values: [[100], ['Åäö中文العربية'], [1]], }, }; - it.each([[dataFrameFromJSON(json), 'test', '"time","name","value"\r\n100,a,1']])( + it.each([[dataFrameFromJSON(json), 'test', '"time","name","value"\r\n100,Åäö中文العربية,1']])( 'should, when logsModel is %s and title is %s, resolve in %s', async (dataFrame, title, expected) => { downloadDataFrameAsCsv(dataFrame, title); @@ -39,22 +39,23 @@ describe('inspector download', () => { const text = await blob.text(); // By default the BOM character should not be included - expect(await hasBOM(blob)).toBe(false); + expect(await getBomType(blob)).toBeUndefined(); expect(text).toEqual(expected); expect(filename).toEqual(`${title}-data-${dateTimeFormat(1400000000000)}.csv`); } ); - it('should include the BOM character when useExcelHeader is true', async () => { - downloadDataFrameAsCsv(dataFrameFromJSON(json), 'test', { useExcelHeader: true }); + it('should use \t as the delimiter and the file should be utf16le if excelCompatibilityMode is true', async () => { + downloadDataFrameAsCsv(dataFrameFromJSON(json), 'test', undefined, undefined, true); const call = (saveAs as unknown as jest.Mock).mock.calls[0]; const blob = call[0]; const filename = call[1]; const text = await blob.text(); - expect(await hasBOM(blob)).toBe(true); - expect(text).toEqual('sep=,\r\n"time","name","value"\r\n100,a,1'); + expect(await getBomType(blob)).toBe('utf-16le'); + expect(blob.type).toBe('text/csv;charset=utf-16le'); + expect(text).toEqual('"time"\t"name"\t"value"\r\n100\tÅäö中文العربية\t1'); expect(filename).toEqual(`test-data-${dateTimeFormat(1400000000000)}.csv`); }); }); @@ -127,18 +128,28 @@ describe('inspector download', () => { }); }); -async function hasBOM(blob: Blob) { +async function getBomType(blob: Blob): Promise<'utf-8' | 'utf-16le' | undefined> { const reader = new FileReader(); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { reader.onload = (event: ProgressEvent) => { if (event.target?.result instanceof ArrayBuffer) { const arr = new Uint8Array(event.target.result); - resolve(arr[0] === 0xef && arr[1] === 0xbb && arr[2] === 0xbf); // Check for UTF-8 BOM + // UTF-8: EF BB BF + if (arr.length >= 3 && arr[0] === 0xef && arr[1] === 0xbb && arr[2] === 0xbf) { + resolve('utf-8'); + return; + } + // UTF-16 LE: FF FE + if (arr.length >= 2 && arr[0] === 0xff && arr[1] === 0xfe) { + resolve('utf-16le'); + return; + } + resolve(undefined); } else { reject(new Error('Unexpected FileReader result type')); } }; reader.onerror = reject; - reader.readAsArrayBuffer(blob.slice(0, 3)); // Read only the first 3 bytes + reader.readAsArrayBuffer(blob.slice(0, 3)); // Read first 3 bytes (covers UTF-8 and UTF-16LE) }); } diff --git a/public/app/features/inspector/utils/download.ts b/public/app/features/inspector/utils/download.ts index d5eeadb95ca..1332180acf2 100644 --- a/public/app/features/inspector/utils/download.ts +++ b/public/app/features/inspector/utils/download.ts @@ -54,14 +54,32 @@ export function downloadDataFrameAsCsv( dataFrame: DataFrame, title: string, csvConfig?: CSVConfig, - transformId: DataTransformerID = DataTransformerID.noop + transformId: DataTransformerID = DataTransformerID.noop, + excelCompatibilityMode = false ) { - const dataFrameCsv = toCSV([dataFrame], csvConfig); - const bomChar = csvConfig?.useExcelHeader ? String.fromCharCode(0xfeff) : ''; + let blob; - const blob = new Blob([bomChar, dataFrameCsv], { - type: 'text/csv;charset=utf-8', - }); + if (excelCompatibilityMode) { + /** + * This compatibility mode creates a utf16le csv file that uses \t as the delimiter. + * This is to fix an issue where excel does not recognize the BOM indicating UTF-8 when the SEP= meta data header is present. + * Without the SEP= metadata header excel will try to use the system list separator. + * If the CSV was created on a system where the separator was ',' it will not work on a system where the separator is ';' + * This is common on locales where ',' is the decimal separator. + * + * When excel opens a utf16le csv file it will no longer try to use the system list separator, and instead use \t as the separator. + */ + const dataFrameCsv = toCSV([dataFrame], { ...csvConfig, useExcelHeader: false, delimiter: '\t' }); + const utf16le = new Uint16Array(Array.from('\ufeff' + dataFrameCsv).map((char) => char.charCodeAt(0))); + blob = new Blob([utf16le], { + type: 'text/csv;charset=utf-16le', + }); + } else { + const dataFrameCsv = toCSV([dataFrame], csvConfig); + blob = new Blob([dataFrameCsv], { + type: 'text/csv;charset=utf-8', + }); + } const transformation = transformId !== DataTransformerID.noop ? '-as-' + transformId.toLocaleLowerCase() : ''; const fileName = `${title}-data${transformation}-${dateTimeFormat(new Date())}.csv`; diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index de6aeba30cc..35074387140 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -4936,12 +4936,11 @@ "dataframe-aria-label": "Select dataframe", "dataframe-label": "Show data frame", "download-csv": "Download CSV", - "download-excel-description": "Adds header to CSV for use with Excel", - "download-excel-label": "Download for Excel", "download-logs": "Download logs", "download-service": "Download service graph", "download-traces": "Download traces", - "excel-header": "Excel header", + "excel-compatibility-mode-description": "Generates a CSV file that's compatible with most Excel versions", + "excel-compatibility-mode-label": "Download for Excel", "formatted": "Formatted data", "formatted-data-description": "Table data is formatted with options defined in the Field and Override tabs.", "formatted-data-label": "Formatted data",