public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Live: keep stream history on 'refresh' (#41492)

Browse Source 
Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Artur Wierzbicki
2021-12-15 11:01:55 -08:00
committed by GitHub
co-authored by Ryan McKinley
parent bb3b5c10e7
commit 84ae13fe5d
21 changed files with 2060 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/grafana-data/src/dataframe/StreamingDataFrame.test.ts
-537
View File
@@ -1,537 +0,0 @@
import { reduceField, ReducerID } from '..';
import { getFieldDisplayName } from '../field';
import { DataFrame, FieldType } from '../types/dataFrame';
import { DataFrameJSON } from './DataFrameJSON';
import { StreamingDataFrame } from './StreamingDataFrame';
describe('Streaming JSON', () => {
describe('when called with a DataFrame', () => {
const json: DataFrameJSON = {
schema: {
fields: [
{ name: 'time', type: FieldType.time },
{ name: 'name', type: FieldType.string },
{ name: 'value', type: FieldType.number },
],
},
data: {
values: [
[100, 200, 300],
['a', 'b', 'c'],
[1, 2, 3],
],
},
};
const stream = new StreamingDataFrame(json, {
maxLength: 5,
maxDelta: 300,
});
it('should create frame with schema & data', () => {
expect(stream.fields.map((f) => ({ name: f.name, value: f.values.buffer }))).toMatchInlineSnapshot(`
Array [
Object {
"name": "time",
"value": Array [
100,
200,
300,
],
},
Object {
"name": "name",
"value": Array [
"a",
"b",
"c",
],
},
Object {
"name": "value",
"value": Array [
1,
2,
3,
],
},
]
`);
});
it('should append new data to frame', () => {
stream.push({
data: {
values: [[400], ['d'], [4]],
},
});
expect(stream.fields.map((f) => ({ name: f.name, value: f.values.buffer }))).toMatchInlineSnapshot(`
Array [
Object {
"name": "time",
"value": Array [
100,
200,
300,
400,
],
},
Object {
"name": "name",
"value": Array [
"a",
"b",
"c",
"d",
],
},
Object {
"name": "value",
"value": Array [
1,
2,
3,
4,
],
},
]
`);
});
it('should append new data and slice based on maxDelta', () => {
stream.push({
data: {
values: [[500], ['e'], [5]],
},
});
expect(stream.fields.map((f) => ({ name: f.name, value: f.values.buffer }))).toMatchInlineSnapshot(`
Array [
Object {
"name": "time",
"value": Array [
200,
300,
400,
500,
],
},
Object {
"name": "name",
"value": Array [
"b",
"c",
"d",
"e",
],
},
Object {
"name": "value",
"value": Array [
2,
3,
4,
5,
],
},
]
`);
});
it('should append new data and slice based on maxLength', () => {
stream.push({
data: {
values: [
[501, 502, 503],
['f', 'g', 'h'],
[6, 7, 8, 9],
],
},
});
expect(stream.fields.map((f) => ({ name: f.name, value: f.values.buffer }))).toMatchInlineSnapshot(`
Array [
Object {
"name": "time",
"value": Array [
400,
500,
501,
502,
503,
],
},
Object {
"name": "name",
"value": Array [
"d",
"e",
"f",
"g",
"h",
],
},
Object {
"name": "value",
"value": Array [
4,
5,
6,
7,
8,
9,
],
},
]
`);
});
});
describe('lengths property is accurate', () => {
const stream = new StreamingDataFrame(
{
schema: {
fields: [{ name: 'simple', type: FieldType.number }],
},
data: {
values: [[100]],
},
},
{
maxLength: 5,
}
);
let val = reduceField({ field: stream.fields[0], reducers: [ReducerID.lastNotNull] })[ReducerID.lastNotNull];
expect(val).toEqual(100);
expect(stream.length).toEqual(1);
stream.push({
data: { values: [[200]] },
});
val = reduceField({ field: stream.fields[0], reducers: [ReducerID.lastNotNull] })[ReducerID.lastNotNull];
expect(val).toEqual(200);
expect(stream.length).toEqual(2);
const copy = ({ ...stream } as any) as DataFrame;
expect(copy.length).toEqual(2);
});
describe('streaming labels column', () => {
const stream = new StreamingDataFrame(
{
schema: {
fields: [
{ name: 'labels', type: FieldType.string },
{ name: 'time', type: FieldType.time },
{ name: 'speed', type: FieldType.number },
{ name: 'light', type: FieldType.number },
],
},
},
{
maxLength: 4,
}
);
stream.push({
data: {
values: [
['sensor=A', 'sensor=B'],
[100, 100],
[10, 15],
[1, 2],
],
},
});
stream.push({
data: {
values: [
['sensor=B', 'sensor=C'],
[200, 200],
[20, 25],
[3, 4],
],
},
});
stream.push({
data: {
values: [
['sensor=A', 'sensor=C'],
[300, 400],
[30, 40],
[5, 6],
],
},
});
expect(stream.fields.map((f) => ({ name: f.name, labels: f.labels, values: f.values.buffer })))
.toMatchInlineSnapshot(`
Array [
Object {
"labels": undefined,
"name": "time",
"values": Array [
100,
200,
300,
400,
],
},
Object {
"labels": Object {
"sensor": "A",
},
"name": "speed",
"values": Array [
10,
undefined,
30,
undefined,
],
},
Object {
"labels": Object {
"sensor": "A",
},
"name": "light",
"values": Array [
1,
undefined,
5,
undefined,
],
},
Object {
"labels": Object {
"sensor": "B",
},
"name": "speed",
"values": Array [
15,
20,
undefined,
undefined,
],
},
Object {
"labels": Object {
"sensor": "B",
},
"name": "light",
"values": Array [
2,
3,
undefined,
undefined,
],
},
Object {
"labels": Object {
"sensor": "C",
},
"name": "speed",
"values": Array [
undefined,
25,
undefined,
40,
],
},
Object {
"labels": Object {
"sensor": "C",
},
"name": "light",
"values": Array [
undefined,
4,
undefined,
6,
],
},
]
`);
// Push value with empty labels
stream.push({
data: {
values: [[''], [500], [50], [7]],
},
});
expect(stream.fields.map((f) => getFieldDisplayName(f, stream, [stream]))).toMatchInlineSnapshot(`
Array [
"time",
"speed A",
"light A",
"speed B",
"light B",
"speed C",
"light C",
"speed 4",
"light 4",
]
`); // speed+light 4 ¯\_(ツ)_/¯ better than undefined labels
});
describe('keep track of packets', () => {
const json: DataFrameJSON = {
schema: {
fields: [
{ name: 'time', type: FieldType.time },
{ name: 'value', type: FieldType.number },
],
},
data: {
values: [
[100, 200, 300],
[1, 2, 3],
],
},
};
const stream = new StreamingDataFrame(json, {
maxLength: 4,
maxDelta: 300,
});
const getSnapshot = (f: StreamingDataFrame) => {
return {
values: f.fields[1].values.toArray(),
info: f.packetInfo,
};
};
expect(getSnapshot(stream)).toMatchInlineSnapshot(`
Object {
"info": Object {
"action": "replace",
"length": 3,
"number": 1,
},
"values": Array [
1,
2,
3,
],
}
`);
stream.push({
data: {
values: [
[400, 500],
[4, 5],
],
},
});
expect(getSnapshot(stream)).toMatchInlineSnapshot(`
Object {
"info": Object {
"action": "append",
"length": 2,
"number": 2,
},
"values": Array [
2,
3,
4,
5,
],
}
`);
stream.push({
data: {
values: [[600], [6]],
},
});
expect(getSnapshot(stream)).toMatchInlineSnapshot(`
Object {
"info": Object {
"action": "append",
"length": 1,
"number": 3,
},
"values": Array [
3,
4,
5,
6,
],
}
`);
});
/*
describe('transpose vertical records', () => {
let vrecsA = [
['sensor=A', 'sensor=B'],
[100, 100],
[10, 15],
];
let vrecsB = [
['sensor=B', 'sensor=C'],
[200, 200],
[20, 25],
];
let vrecsC = [
['sensor=A', 'sensor=C'],
[300, 400],
[30, 40],
];
let cTables = transpose(vrecsC);
expect(cTables).toMatchInlineSnapshot(`
Array [
Array [
"sensor=A",
"sensor=C",
],
Array [
Array [
Array [
300,
],
Array [
30,
],
],
Array [
Array [
400,
],
Array [
40,
],
],
],
]
`);
let cJoined = join(cTables[1]);
expect(cJoined).toMatchInlineSnapshot(`
Array [
Array [
300,
400,
],
Array [
30,
undefined,
],
Array [
undefined,
40,
],
]
`);
});
*/
});
packages/grafana-data/src/dataframe/StreamingDataFrame.ts
-365
View File
@@ -1,365 +0,0 @@
import { Field, DataFrame, FieldType, Labels, QueryResultMeta } from '../types';
import { ArrayVector } from '../vector';
import { DataFrameJSON, decodeFieldValueEntities, FieldSchema } from './DataFrameJSON';
import { guessFieldTypeFromValue } from './processDataFrame';
import { join } from '../transformations/transformers/joinDataFrames';
import { AlignedData } from 'uplot';
/**
* Indicate if the frame is appened or replace
*
* @public -- but runtime
*/
export enum StreamingFrameAction {
Append = 'append',
Replace = 'replace',
}
/**
* Stream packet info is attached to StreamingDataFrames and indicate how many
* rows were added to the end of the frame. The number of discarded rows can be
* calculated from previous state
*
* @public -- but runtime
*/
export interface StreamPacketInfo {
number: number;
action: StreamingFrameAction;
length: number;
}
/**
* @alpha
*/
export interface StreamingFrameOptions {
maxLength?: number; // 1000
maxDelta?: number; // how long to keep things
action?: StreamingFrameAction; // default will append
}
enum PushMode {
wide,
labels,
// long
}
/**
* Unlike a circular buffer, this will append and periodically slice the front
*
* @alpha
*/
export class StreamingDataFrame implements DataFrame {
name?: string;
refId?: string;
meta: QueryResultMeta = {};
fields: Array<Field<any, ArrayVector<any>>> = [];
length = 0;
options: StreamingFrameOptions;
private schemaFields: FieldSchema[] = [];
private timeFieldIndex = -1;
private pushMode = PushMode.wide;
private alwaysReplace = false;
// current labels
private labels: Set<string> = new Set();
readonly packetInfo: StreamPacketInfo = {
number: 0,
action: StreamingFrameAction.Replace,
length: 0,
};
constructor(frame: DataFrameJSON, opts?: StreamingFrameOptions) {
this.options = {
maxLength: 1000,
maxDelta: Infinity,
...opts,
};
this.alwaysReplace = this.options.action === StreamingFrameAction.Replace;
this.push(frame);
}
/**
* apply the new message to the existing data. This will replace the existing schema
* if a new schema is included in the message, or append data matching the current schema
*/
push(msg: DataFrameJSON) {
const { schema, data } = msg;
this.packetInfo.number++;
if (schema) {
this.pushMode = PushMode.wide;
this.timeFieldIndex = schema.fields.findIndex((f) => f.type === FieldType.time);
if (
this.timeFieldIndex === 1 &&
schema.fields[0].name === 'labels' &&
schema.fields[0].type === FieldType.string
) {
this.pushMode = PushMode.labels;
this.timeFieldIndex = 0; // after labels are removed!
}
const niceSchemaFields = this.pushMode === PushMode.labels ? schema.fields.slice(1) : schema.fields;
this.refId = schema.refId;
if (schema.meta) {
this.meta = { ...schema.meta };
}
if (hasSameStructure(this.schemaFields, niceSchemaFields)) {
const len = niceSchemaFields.length;
this.fields.forEach((f, idx) => {
const sf = niceSchemaFields[idx % len];
f.config = sf.config ?? {};
f.labels = sf.labels;
});
} else {
const isWide = this.pushMode === PushMode.wide;
this.fields = niceSchemaFields.map((f) => {
return {
config: f.config ?? {},
name: f.name,
labels: f.labels,
type: f.type ?? FieldType.other,
// transfer old values by type & name, unless we relied on labels to match fields
values: isWide
? this.fields.find((of) => of.name === f.name && f.type === of.type)?.values ?? new ArrayVector()
: new ArrayVector(),
};
});
}
this.schemaFields = niceSchemaFields;
}
if (data && data.values.length && data.values[0].length) {
let { values, entities } = data;
if (entities) {
entities.forEach((ents, i) => {
if (ents) {
decodeFieldValueEntities(ents, values[i]);
// TODO: append replacements to field
}
});
}
if (this.pushMode === PushMode.labels) {
// augment and transform data to match current schema for standard circPush() path
const labeledTables = transpose(values);
// make sure fields are initalized for each label
for (const label of labeledTables.keys()) {
if (!this.labels.has(label)) {
this.addLabel(label);
}
}
// TODO: cache higher up
let dummyTable = Array(this.schemaFields.length).fill([]);
let tables: AlignedData[] = [];
this.labels.forEach((label) => {
tables.push(labeledTables.get(label) ?? dummyTable);
});
values = join(tables);
}
if (values.length !== this.fields.length) {
if (this.fields.length) {
throw new Error(
`push message mismatch. Expected: ${this.fields.length}, recieved: ${values.length} (labels=${
this.pushMode === PushMode.labels
})`
);
}
this.fields = values.map((vals, idx) => {
let name = `Field ${idx}`;
let type = guessFieldTypeFromValue(vals[0]);
const isTime = idx === 0 && type === FieldType.number && vals[0] > 1600016688632;
if (isTime) {
type = FieldType.time;
name = 'Time';
}
return {
name,
type,
config: {},
values: new ArrayVector([]),
};
});
}
let appended = values;
this.packetInfo.length = values[0].length;
if (this.alwaysReplace || !this.length) {
this.packetInfo.action = StreamingFrameAction.Replace;
} else {
this.packetInfo.action = StreamingFrameAction.Append;
// mutates appended
appended = this.fields.map((f) => f.values.buffer);
circPush(appended, values, this.options.maxLength, this.timeFieldIndex, this.options.maxDelta);
}
appended.forEach((v, i) => {
const { state, values } = this.fields[i];
values.buffer = v;
if (state) {
state.calcs = undefined;
}
});
// Update the frame length
this.length = appended[0].length;
}
}
// adds a set of fields for a new label
private addLabel(label: string) {
let labelCount = this.labels.size;
// parse labels
const parsedLabels: Labels = {};
if (label.length) {
label.split(',').forEach((kv) => {
const [key, val] = kv.trim().split('=');
parsedLabels[key] = val;
});
}
if (labelCount === 0) {
// mutate existing fields and add labels
this.fields.forEach((f, i) => {
if (i > 0) {
f.labels = parsedLabels;
}
});
} else {
for (let i = 1; i < this.schemaFields.length; i++) {
let proto = this.schemaFields[i] as Field;
this.fields.push({
...proto,
config: proto.config ?? {},
labels: parsedLabels,
values: new ArrayVector(Array(this.length).fill(undefined)),
});
}
}
this.labels.add(label);
}
}
// converts vertical insertion records with table keys in [0] and column values in [1...N]
// to join()-able tables with column arrays
export function transpose(vrecs: any[][]) {
let tableKeys = new Set(vrecs[0]);
let tables = new Map();
tableKeys.forEach((key) => {
let cols = Array(vrecs.length - 1)
.fill(null)
.map(() => []);
tables.set(key, cols);
});
for (let r = 0; r < vrecs[0].length; r++) {
let table = tables.get(vrecs[0][r]);
for (let c = 1; c < vrecs.length; c++) {
table[c - 1].push(vrecs[c][r]);
}
}
return tables;
}
// binary search for index of closest value
function closestIdx(num: number, arr: number[], lo?: number, hi?: number) {
let mid;
lo = lo || 0;
hi = hi || arr.length - 1;
let bitwise = hi <= 2147483647;
while (hi - lo > 1) {
mid = bitwise ? (lo + hi) >> 1 : Math.floor((lo + hi) / 2);
if (arr[mid] < num) {
lo = mid;
} else {
hi = mid;
}
}
if (num - arr[lo] <= arr[hi] - num) {
return lo;
}
return hi;
}
/**
* @internal // not exported in yet
*/
export function getLastStreamingDataFramePacket(frame: DataFrame) {
const pi = (frame as StreamingDataFrame).packetInfo;
return pi?.action ? pi : undefined;
}
// mutable circular push
function circPush(data: number[][], newData: number[][], maxLength = Infinity, deltaIdx = 0, maxDelta = Infinity) {
for (let i = 0; i < data.length; i++) {
data[i] = data[i].concat(newData[i]);
}
const nlen = data[0].length;
let sliceIdx = 0;
if (nlen > maxLength) {
sliceIdx = nlen - maxLength;
}
if (maxDelta !== Infinity && deltaIdx >= 0) {
const deltaLookup = data[deltaIdx];
const low = deltaLookup[sliceIdx];
const high = deltaLookup[nlen - 1];
if (high - low > maxDelta) {
sliceIdx = closestIdx(high - maxDelta, deltaLookup, sliceIdx);
}
}
if (sliceIdx) {
for (let i = 0; i < data.length; i++) {
data[i] = data[i].slice(sliceIdx);
}
}
return sliceIdx;
}
function hasSameStructure(a: FieldSchema[], b: FieldSchema[]): boolean {
if (a?.length !== b.length) {
return false;
}
for (let i = 0; i < a.length; i++) {
const fA = a[i];
const fB = b[i];
if (fA.name !== fB.name || fA.type !== fB.type) {
return false;
}
}
return true;
}
packages/grafana-data/src/dataframe/index.ts
-1
View File
@@ -6,6 +6,5 @@ export * from './processDataFrame';
export * from './dimensions';
export * from './ArrayDataFrame';
export * from './DataFrameJSON';
export { StreamingDataFrame, StreamingFrameOptions, StreamingFrameAction } from './StreamingDataFrame';
export * from './frameComparisons';
export { anySeriesWithTimeField } from './utils';
packages/grafana-data/src/dataframe/processDataFrame.ts
+6 -1
View File
@@ -470,7 +470,12 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] {
* Returns a copy that does not include functions
*/
export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
const fields: FieldDTO[] = data.fields.map((f) => {
return toFilteredDataFrameDTO(data);
}
export function toFilteredDataFrameDTO(data: DataFrame, fieldPredicate?: (f: Field) => boolean): DataFrameDTO {
const filteredFields = fieldPredicate ? data.fields.filter(fieldPredicate) : data.fields;
const fields: FieldDTO[] = filteredFields.map((f) => {
let values = f.values.toArray();
// The byte buffers serialize like objects
if (values instanceof Float64Array) {
packages/grafana-data/src/types/live.ts
+6 -1
View File
@@ -125,6 +125,11 @@ export interface LiveChannelPresenceStatus {
users: any; // @alpha -- experimental -- will be filled in when we improve the UI
}
/**
* @alpha -- experimental
*/
export type LiveChannelId = string;
/**
* @alpha -- experimental
*/
@@ -174,7 +179,7 @@ export function isValidLiveChannelAddress(addr?: LiveChannelAddress): addr is Li
*
* @alpha -- experimental
*/
export function toLiveChannelId(addr: LiveChannelAddress): string {
export function toLiveChannelId(addr: LiveChannelAddress): LiveChannelId {
if (!addr.scope) {
return '';
}
packages/grafana-runtime/src/services/live.ts
+20 -2
View File
@@ -4,7 +4,6 @@ import {
LiveChannelAddress,
LiveChannelEvent,
LiveChannelPresenceStatus,
StreamingFrameOptions,
} from '@grafana/data';
import { Observable } from 'rxjs';
@@ -15,6 +14,25 @@ export interface LiveDataFilter {
fields?: string[];
}
/**
* Indicate if the frame is appened or replace
*
* @alpha
*/
export enum StreamingFrameAction {
Append = 'append',
Replace = 'replace',
}
/**
* @alpha
*/
export interface StreamingFrameOptions {
maxLength: number; // 1000
maxDelta: number; // how long to keep things
action: StreamingFrameAction; // default will append
}
/**
* @alpha
*/
@@ -22,7 +40,7 @@ export interface LiveDataStreamOptions {
addr: LiveChannelAddress;
frame?: DataFrame; // initial results
key?: string;
buffer?: StreamingFrameOptions;
buffer?: Partial<StreamingFrameOptions>;
filter?: LiveDataFilter;
}
packages/grafana-runtime/src/utils/DataSourceWithBackend.ts
+12 -8
View File
@@ -9,14 +9,18 @@ import {
makeClassES5Compatible,
DataFrame,
parseLiveChannelAddress,
StreamingFrameOptions,
StreamingFrameAction,
getDataSourceRef,
DataSourceRef,
} from '@grafana/data';
import { merge, Observable, of } from 'rxjs';
import { catchError, switchMap } from 'rxjs/operators';
import { getBackendSrv, getDataSourceSrv, getGrafanaLiveSrv } from '../services';
import {
getBackendSrv,
getDataSourceSrv,
getGrafanaLiveSrv,
StreamingFrameOptions,
StreamingFrameAction,
} from '../services';
import { BackendDataSourceResponse, toDataQueryResponse } from './queryResponse';
/**
@@ -250,7 +254,7 @@ class DataSourceWithBackend<
export function toStreamingDataResponse<TQuery extends DataQuery = DataQuery>(
rsp: DataQueryResponse,
req: DataQueryRequest<TQuery>,
getter: (req: DataQueryRequest<TQuery>, frame: DataFrame) => StreamingFrameOptions
getter: (req: DataQueryRequest<TQuery>, frame: DataFrame) => Partial<StreamingFrameOptions>
): Observable<DataQueryResponse> {
const live = getGrafanaLiveSrv();
if (!live) {
@@ -291,22 +295,22 @@ export function toStreamingDataResponse<TQuery extends DataQuery = DataQuery>(
export type StreamOptionsProvider<TQuery extends DataQuery = DataQuery> = (
request: DataQueryRequest<TQuery>,
frame: DataFrame
) => StreamingFrameOptions;
) => Partial<StreamingFrameOptions>;
/**
* @public
*/
export const standardStreamOptionsProvider: StreamOptionsProvider = (request: DataQueryRequest, frame: DataFrame) => {
const buffer: StreamingFrameOptions = {
const opts: Partial<StreamingFrameOptions> = {
maxLength: request.maxDataPoints ?? 500,
action: StreamingFrameAction.Append,
};
// For recent queries, clamp to the current time range
if (request.rangeRaw?.to === 'now') {
buffer.maxDelta = request.range.to.valueOf() - request.range.from.valueOf();
opts.maxDelta = request.range.to.valueOf() - request.range.from.valueOf();
}
return buffer;
return opts;
};
//@ts-ignore