public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create unique ids in Loki

Browse Source
This commit is contained in:
Ivana
2020-12-11 17:49:24 +01:00
parent 03b667c160
commit 2b8cd724d1
3 changed files with 96 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-ui/src/components/Logs/LogRows.tsx
+2 -2
View File
@@ -137,7 +137,7 @@ class UnThemedLogRows extends PureComponent<Props, State> {
{hasData &&
firstRows.map((row, index) => (
<LogRow
key={`${row.uid}-${index}`}
key={row.uid}
getRows={getRows}
getRowContext={getRowContext}
highlighterExpressions={highlighterExpressions}
@@ -162,7 +162,7 @@ class UnThemedLogRows extends PureComponent<Props, State> {
renderAll &&
lastRows.map((row, index) => (
<LogRow
key={`${row.uid}-${index}`}
key={row.uid}
getRows={getRows}
getRowContext={getRowContext}
row={row}
public/app/plugins/datasource/loki/result_transformer.test.ts
+68
View File
@@ -71,6 +71,37 @@ describe('loki result transformer', () => {
expect(data[1].fields[1].values.get(0)).toEqual(streamResult[1].values[0][1]);
expect(data[1].fields[2].values.get(0)).toEqual('75d73d66cff40f9d1a1f2d5a0bf295d0');
});
it('should always generate unique ids for logs', () => {
const streamResultWithDuplicateLogs: LokiStreamResult[] = [
{
stream: {
foo: 'bar',
},
values: [
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'],
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'],
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-duplicated"'],
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'],
],
},
{
stream: {
bar: 'foo',
},
values: [['1579857562021617000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-dupliicated"']],
},
];
const data = streamResultWithDuplicateLogs.map(stream => ResultTransformer.lokiStreamResultToDataFrame(stream));
expect(data[0].fields[2].values.get(0)).toEqual('65cee200875f58ee1430d8bd2e8b74e7');
expect(data[0].fields[2].values.get(1)).toEqual('65cee200875f58ee1430d8bd2e8b74e7_1');
expect(data[0].fields[2].values.get(2)).not.toEqual('65cee200875f58ee1430d8bd2e8b74e7_2');
expect(data[0].fields[2].values.get(3)).toEqual('65cee200875f58ee1430d8bd2e8b74e7_2');
expect(data[1].fields[2].values.get(0)).not.toEqual('65cee200875f58ee1430d8bd2e8b74e7_3');
});
});
describe('lokiStreamsToDataFrames', () => {
@@ -131,7 +162,44 @@ describe('loki result transformer', () => {
id: '19e8e093d70122b3b53cb6e24efd6e2d',
});
});
it('should always generate unique ids for logs', () => {
const tailResponse: LokiTailResponse = {
streams: [
{
stream: {
filename: '/var/log/grafana/grafana.log',
job: 'grafana',
},
values: [
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 2"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Not dupplicated"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 2"'],
],
},
],
};
const data = new CircularDataFrame({ capacity: 6 });
data.addField({ name: 'ts', type: FieldType.time, config: { displayName: 'Time' } });
data.addField({ name: 'tsNs', type: FieldType.time, config: { displayName: 'Time ns' } });
data.addField({ name: 'line', type: FieldType.string }).labels = { job: 'grafana' };
data.addField({ name: 'labels', type: FieldType.other });
data.addField({ name: 'id', type: FieldType.string });
ResultTransformer.appendResponseToBufferedData(tailResponse, data);
expect(data.get(0).id).toEqual('870e4d105741bdfc2c67904ee480d4f3');
expect(data.get(1).id).toEqual('870e4d105741bdfc2c67904ee480d4f3_1');
expect(data.get(2).id).toEqual('707e4ec2b842f389dbb993438505856d');
expect(data.get(3).id).toEqual('78f044015a58fad3e257a855b167d85e');
expect(data.get(4).id).toEqual('870e4d105741bdfc2c67904ee480d4f3_2');
expect(data.get(5).id).toEqual('707e4ec2b842f389dbb993438505856d_1');
});
});
describe('createMetricLabel', () => {
it('should create correct label based on passed variables', () => {
const label = ResultTransformer.createMetricLabel({}, ({
public/app/plugins/datasource/loki/result_transformer.ts
+26 -4
View File
@@ -53,12 +53,15 @@ export function lokiStreamResultToDataFrame(stream: LokiStreamResult, reverse?:
const lines = new ArrayVector<string>([]);
const uids = new ArrayVector<string>([]);
// We need to store and track all used uids to ensure that uids are unique
const usedUids: { string?: number } = {};
for (const [ts, line] of stream.values) {
// num ns epoch in string, we convert it to iso string here so it matches old format
times.add(new Date(parseInt(ts.substr(0, ts.length - 6), 10)).toISOString());
timesNs.add(ts);
lines.add(line);
uids.add(createUid(ts, labelsString, line));
uids.add(createUid(ts, labelsString, line, usedUids));
}
return constructDataFrame(times, timesNs, lines, uids, labels, reverse, refId);
@@ -127,6 +130,9 @@ export function appendResponseToBufferedData(response: LokiTailResponse, data: M
const labelsField = data.fields[3];
const idField = data.fields[4];
// We need to store and track all used uids to ensure that uids are unique
const usedUids: { string?: number } = {};
for (const stream of streams) {
// Find unique labels
const unique = findUniqueLabels(stream.stream, baseLabels);
@@ -141,13 +147,29 @@ export function appendResponseToBufferedData(response: LokiTailResponse, data: M
tsNsField.values.add(ts);
lineField.values.add(line);
labelsField.values.add(unique);
idField.values.add(createUid(ts, allLabelsString, line));
idField.values.add(createUid(ts, allLabelsString, line, usedUids));
}
}
}
function createUid(ts: string, labelsString: string, line: string): string {
return md5(`${ts}_${labelsString}_${line}`);
function createUid(ts: string, labelsString: string, line: string, usedUids: any): string {
// Generate id as hashed nanosecond timestamp, labels and line (this does not have to be unique)
let id = md5(`${ts}_${labelsString}_${line}`);
// Check if generated id is unique
// If not and we've already used it, append it's count after it
if (id in usedUids) {
// Increase the count
const newCount = usedUids[id] + 1;
usedUids[id] = newCount;
// Append index to generated id to make it unique
id = `${id}_${newCount}`;
} else {
// If id is unique and wasn't used, add it to allUids and start count at 0
usedUids[id] = 0;
}
// Return unique id
return id;
}
function lokiMatrixToTimeSeries(matrixResult: LokiMatrixResult, options: TransformerOptions): TimeSeries {