diff --git a/packages/grafana-ui/src/components/Logs/LogRows.tsx b/packages/grafana-ui/src/components/Logs/LogRows.tsx index 946c62866c1..91fee6a6804 100644 --- a/packages/grafana-ui/src/components/Logs/LogRows.tsx +++ b/packages/grafana-ui/src/components/Logs/LogRows.tsx @@ -137,7 +137,7 @@ class UnThemedLogRows extends PureComponent { {hasData && firstRows.map((row, index) => ( { renderAll && lastRows.map((row, index) => ( { expect(data[1].fields[1].values.get(0)).toEqual(streamResult[1].values[0][1]); expect(data[1].fields[2].values.get(0)).toEqual('75d73d66cff40f9d1a1f2d5a0bf295d0'); }); + + it('should always generate unique ids for logs', () => { + const streamResultWithDuplicateLogs: LokiStreamResult[] = [ + { + stream: { + foo: 'bar', + }, + + values: [ + ['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'], + ['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'], + ['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-duplicated"'], + ['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'], + ], + }, + { + stream: { + bar: 'foo', + }, + values: [['1579857562021617000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-dupliicated"']], + }, + ]; + + const data = streamResultWithDuplicateLogs.map(stream => ResultTransformer.lokiStreamResultToDataFrame(stream)); + + expect(data[0].fields[2].values.get(0)).toEqual('65cee200875f58ee1430d8bd2e8b74e7'); + expect(data[0].fields[2].values.get(1)).toEqual('65cee200875f58ee1430d8bd2e8b74e7_1'); + expect(data[0].fields[2].values.get(2)).not.toEqual('65cee200875f58ee1430d8bd2e8b74e7_2'); + expect(data[0].fields[2].values.get(3)).toEqual('65cee200875f58ee1430d8bd2e8b74e7_2'); + expect(data[1].fields[2].values.get(0)).not.toEqual('65cee200875f58ee1430d8bd2e8b74e7_3'); + }); }); describe('lokiStreamsToDataFrames', () => { @@ -131,7 +162,44 @@ describe('loki result transformer', () => { id: '19e8e093d70122b3b53cb6e24efd6e2d', }); }); + + it('should always generate unique ids for logs', () => { + const tailResponse: LokiTailResponse = { + streams: [ + { + stream: { + filename: '/var/log/grafana/grafana.log', + job: 'grafana', + }, + values: [ + ['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'], + ['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'], + ['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 2"'], + ['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Not dupplicated"'], + ['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'], + ['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 2"'], + ], + }, + ], + }; + + const data = new CircularDataFrame({ capacity: 6 }); + data.addField({ name: 'ts', type: FieldType.time, config: { displayName: 'Time' } }); + data.addField({ name: 'tsNs', type: FieldType.time, config: { displayName: 'Time ns' } }); + data.addField({ name: 'line', type: FieldType.string }).labels = { job: 'grafana' }; + data.addField({ name: 'labels', type: FieldType.other }); + data.addField({ name: 'id', type: FieldType.string }); + + ResultTransformer.appendResponseToBufferedData(tailResponse, data); + expect(data.get(0).id).toEqual('870e4d105741bdfc2c67904ee480d4f3'); + expect(data.get(1).id).toEqual('870e4d105741bdfc2c67904ee480d4f3_1'); + expect(data.get(2).id).toEqual('707e4ec2b842f389dbb993438505856d'); + expect(data.get(3).id).toEqual('78f044015a58fad3e257a855b167d85e'); + expect(data.get(4).id).toEqual('870e4d105741bdfc2c67904ee480d4f3_2'); + expect(data.get(5).id).toEqual('707e4ec2b842f389dbb993438505856d_1'); + }); }); + describe('createMetricLabel', () => { it('should create correct label based on passed variables', () => { const label = ResultTransformer.createMetricLabel({}, ({ diff --git a/public/app/plugins/datasource/loki/result_transformer.ts b/public/app/plugins/datasource/loki/result_transformer.ts index b79ae0178db..1b5c6e79b74 100644 --- a/public/app/plugins/datasource/loki/result_transformer.ts +++ b/public/app/plugins/datasource/loki/result_transformer.ts @@ -53,12 +53,15 @@ export function lokiStreamResultToDataFrame(stream: LokiStreamResult, reverse?: const lines = new ArrayVector([]); const uids = new ArrayVector([]); + // We need to store and track all used uids to ensure that uids are unique + const usedUids: { string?: number } = {}; + for (const [ts, line] of stream.values) { // num ns epoch in string, we convert it to iso string here so it matches old format times.add(new Date(parseInt(ts.substr(0, ts.length - 6), 10)).toISOString()); timesNs.add(ts); lines.add(line); - uids.add(createUid(ts, labelsString, line)); + uids.add(createUid(ts, labelsString, line, usedUids)); } return constructDataFrame(times, timesNs, lines, uids, labels, reverse, refId); @@ -127,6 +130,9 @@ export function appendResponseToBufferedData(response: LokiTailResponse, data: M const labelsField = data.fields[3]; const idField = data.fields[4]; + // We need to store and track all used uids to ensure that uids are unique + const usedUids: { string?: number } = {}; + for (const stream of streams) { // Find unique labels const unique = findUniqueLabels(stream.stream, baseLabels); @@ -141,13 +147,29 @@ export function appendResponseToBufferedData(response: LokiTailResponse, data: M tsNsField.values.add(ts); lineField.values.add(line); labelsField.values.add(unique); - idField.values.add(createUid(ts, allLabelsString, line)); + idField.values.add(createUid(ts, allLabelsString, line, usedUids)); } } } -function createUid(ts: string, labelsString: string, line: string): string { - return md5(`${ts}_${labelsString}_${line}`); +function createUid(ts: string, labelsString: string, line: string, usedUids: any): string { + // Generate id as hashed nanosecond timestamp, labels and line (this does not have to be unique) + let id = md5(`${ts}_${labelsString}_${line}`); + + // Check if generated id is unique + // If not and we've already used it, append it's count after it + if (id in usedUids) { + // Increase the count + const newCount = usedUids[id] + 1; + usedUids[id] = newCount; + // Append index to generated id to make it unique + id = `${id}_${newCount}`; + } else { + // If id is unique and wasn't used, add it to allUids and start count at 0 + usedUids[id] = 0; + } + // Return unique id + return id; } function lokiMatrixToTimeSeries(matrixResult: LokiMatrixResult, options: TransformerOptions): TimeSeries {