grafana/data: Reorganise code (#19136)

* Organise data frame and vectors code

* Organise transformations

* Move dataframe utils to dataframe dir

* Organise datetime utils

* Organise text utils

* Organise logs utils

* Revert "Organise logs utils"

This reverts commit c24115c755.

* registry -> Registry

* Transformations reorg

packages/grafana-data/src/vector/AppendedVectors.test.ts

@@ -0,0 +1,23 @@
+import { ArrayVector } from './ArrayVector';
+import { AppendedVectors } from './AppendedVectors';
+
+describe('Check Appending Vector', () => {
+  it('should transparently join them', () => {
+    const appended = new AppendedVectors();
+    appended.append(new ArrayVector([1, 2, 3]));
+    appended.append(new ArrayVector([4, 5, 6]));
+    appended.append(new ArrayVector([7, 8, 9]));
+    expect(appended.length).toEqual(9);
+
+    appended.setLength(5);
+    expect(appended.length).toEqual(5);
+    appended.append(new ArrayVector(['a', 'b', 'c']));
+    expect(appended.length).toEqual(8);
+    expect(appended.toArray()).toEqual([1, 2, 3, 4, 5, 'a', 'b', 'c']);
+
+    appended.setLength(2);
+    appended.setLength(6);
+    appended.append(new ArrayVector(['x', 'y', 'z']));
+    expect(appended.toArray()).toEqual([1, 2, undefined, undefined, undefined, undefined, 'x', 'y', 'z']);
+  });
+});

packages/grafana-data/src/vector/AppendedVectors.ts

@@ -0,0 +1,73 @@
+import { Vector } from '../types/vector';
+import { vectorToArray } from './vectorToArray';
+
+interface AppendedVectorInfo<T> {
+  start: number;
+  end: number;
+  values: Vector<T>;
+}
+
+/**
+ * This may be more trouble than it is worth.  This trades some computation time for
+ * RAM -- rather than allocate a new array the size of all previous arrays, this just
+ * points the correct index to their original array values
+ */
+export class AppendedVectors<T = any> implements Vector<T> {
+  length = 0;
+  source: Array<AppendedVectorInfo<T>> = new Array<AppendedVectorInfo<T>>();
+
+  constructor(startAt = 0) {
+    this.length = startAt;
+  }
+
+  /**
+   * Make the vector look like it is this long
+   */
+  setLength(length: number) {
+    if (length > this.length) {
+      // make the vector longer (filling with undefined)
+      this.length = length;
+    } else if (length < this.length) {
+      // make the array shorter
+      const sources: Array<AppendedVectorInfo<T>> = new Array<AppendedVectorInfo<T>>();
+      for (const src of this.source) {
+        sources.push(src);
+        if (src.end > length) {
+          src.end = length;
+          break;
+        }
+      }
+      this.source = sources;
+      this.length = length;
+    }
+  }
+
+  append(v: Vector<T>): AppendedVectorInfo<T> {
+    const info = {
+      start: this.length,
+      end: this.length + v.length,
+      values: v,
+    };
+    this.length = info.end;
+    this.source.push(info);
+    return info;
+  }
+
+  get(index: number): T {
+    for (let i = 0; i < this.source.length; i++) {
+      const src = this.source[i];
+      if (index >= src.start && index < src.end) {
+        return src.values.get(index - src.start);
+      }
+    }
+    return (undefined as unknown) as T;
+  }
+
+  toArray(): T[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): T[] {
+    return vectorToArray(this);
+  }
+}

packages/grafana-data/src/vector/ArrayVector.ts

@@ -0,0 +1,37 @@
+import { MutableVector } from '../types/vector';
+
+export class ArrayVector<T = any> implements MutableVector<T> {
+  buffer: T[];
+
+  constructor(buffer?: T[]) {
+    this.buffer = buffer ? buffer : [];
+  }
+
+  get length() {
+    return this.buffer.length;
+  }
+
+  add(value: T) {
+    this.buffer.push(value);
+  }
+
+  get(index: number): T {
+    return this.buffer[index];
+  }
+
+  set(index: number, value: T) {
+    this.buffer[index] = value;
+  }
+
+  reverse() {
+    this.buffer.reverse();
+  }
+
+  toArray(): T[] {
+    return this.buffer;
+  }
+
+  toJSON(): T[] {
+    return this.buffer;
+  }
+}

packages/grafana-data/src/vector/CircularVector.test.ts

@@ -0,0 +1,131 @@
+import { CircularVector } from './CircularVector';
+
+describe('Check Circular Vector', () => {
+  it('should append values', () => {
+    const buffer = [1, 2, 3];
+    const v = new CircularVector({ buffer }); // tail is default option
+    expect(v.toArray()).toEqual([1, 2, 3]);
+
+    v.add(4);
+    expect(v.toArray()).toEqual([2, 3, 4]);
+
+    v.add(5);
+    expect(v.toArray()).toEqual([3, 4, 5]);
+
+    v.add(6);
+    expect(v.toArray()).toEqual([4, 5, 6]);
+
+    v.add(7);
+    expect(v.toArray()).toEqual([5, 6, 7]);
+
+    v.add(8);
+    expect(v.toArray()).toEqual([6, 7, 8]);
+  });
+
+  it('should grow buffer until it hits capacity (append)', () => {
+    const v = new CircularVector({ capacity: 3 }); // tail is default option
+    expect(v.toArray()).toEqual([]);
+
+    v.add(1);
+    expect(v.toArray()).toEqual([1]);
+
+    v.add(2);
+    expect(v.toArray()).toEqual([1, 2]);
+
+    v.add(3);
+    expect(v.toArray()).toEqual([1, 2, 3]);
+
+    v.add(4);
+    expect(v.toArray()).toEqual([2, 3, 4]);
+
+    v.add(5);
+    expect(v.toArray()).toEqual([3, 4, 5]);
+  });
+
+  it('should prepend values', () => {
+    const buffer = [3, 2, 1];
+    const v = new CircularVector({ buffer, append: 'head' });
+    expect(v.toArray()).toEqual([3, 2, 1]);
+
+    v.add(4);
+    expect(v.toArray()).toEqual([4, 3, 2]);
+
+    v.add(5);
+    expect(v.toArray()).toEqual([5, 4, 3]);
+
+    v.add(6);
+    expect(v.toArray()).toEqual([6, 5, 4]);
+
+    v.add(7);
+    expect(v.toArray()).toEqual([7, 6, 5]);
+
+    v.add(8);
+    expect(v.toArray()).toEqual([8, 7, 6]);
+  });
+
+  it('should expand buffer and then prepend', () => {
+    const v = new CircularVector({ capacity: 3, append: 'head' });
+    expect(v.toArray()).toEqual([]);
+
+    v.add(1);
+    expect(v.toArray()).toEqual([1]);
+
+    v.add(2);
+    expect(v.toArray()).toEqual([2, 1]);
+
+    v.add(3);
+    expect(v.toArray()).toEqual([3, 2, 1]);
+
+    v.add(4);
+    expect(v.toArray()).toEqual([4, 3, 2]);
+
+    v.add(5);
+    expect(v.toArray()).toEqual([5, 4, 3]);
+  });
+
+  it('should reduce size and keep working (tail)', () => {
+    const buffer = [1, 2, 3, 4, 5];
+    const v = new CircularVector({ buffer });
+    expect(v.toArray()).toEqual([1, 2, 3, 4, 5]);
+
+    v.setCapacity(3);
+    expect(v.toArray()).toEqual([3, 4, 5]);
+
+    v.add(6);
+    expect(v.toArray()).toEqual([4, 5, 6]);
+
+    v.add(7);
+    expect(v.toArray()).toEqual([5, 6, 7]);
+  });
+
+  it('should reduce size and keep working (head)', () => {
+    const buffer = [5, 4, 3, 2, 1];
+    const v = new CircularVector({ buffer, append: 'head' });
+    expect(v.toArray()).toEqual([5, 4, 3, 2, 1]);
+
+    v.setCapacity(3);
+    expect(v.toArray()).toEqual([5, 4, 3]);
+
+    v.add(6);
+    expect(v.toArray()).toEqual([6, 5, 4]);
+
+    v.add(7);
+    expect(v.toArray()).toEqual([7, 6, 5]);
+  });
+
+  it('change buffer direction', () => {
+    const buffer = [1, 2, 3];
+    const v = new CircularVector({ buffer });
+    expect(v.toArray()).toEqual([1, 2, 3]);
+
+    v.setAppendMode('head');
+    expect(v.toArray()).toEqual([3, 2, 1]);
+
+    v.add(4);
+    expect(v.toArray()).toEqual([4, 3, 2]);
+
+    v.setAppendMode('tail');
+    v.add(5);
+    expect(v.toArray()).toEqual([3, 4, 5]);
+  });
+});

packages/grafana-data/src/vector/CircularVector.ts

@@ -0,0 +1,138 @@
+import { MutableVector } from '../types/vector';
+import { vectorToArray } from './vectorToArray';
+
+interface CircularOptions<T> {
+  buffer?: T[];
+  append?: 'head' | 'tail';
+  capacity?: number;
+}
+
+/**
+ * Circular vector uses a single buffer to capture a stream of values
+ * overwriting the oldest value on add.
+ *
+ * This supports addting to the 'head' or 'tail' and will grow the buffer
+ * to match a configured capacity.
+ */
+export class CircularVector<T = any> implements MutableVector<T> {
+  private buffer: T[];
+  private index: number;
+  private capacity: number;
+  private tail: boolean;
+
+  constructor(options: CircularOptions<T>) {
+    this.buffer = options.buffer || [];
+    this.capacity = this.buffer.length;
+    this.tail = 'head' !== options.append;
+    this.index = 0;
+
+    this.add = this.getAddFunction();
+    if (options.capacity) {
+      this.setCapacity(options.capacity);
+    }
+  }
+
+  /**
+   * This gets the appropriate add function depending on the buffer state:
+   *  * head vs tail
+   *  * growing buffer vs overwriting values
+   */
+  private getAddFunction() {
+    // When we are not at capacity, it should actually modify the buffer
+    if (this.capacity > this.buffer.length) {
+      if (this.tail) {
+        return (value: T) => {
+          this.buffer.push(value);
+          if (this.buffer.length >= this.capacity) {
+            this.add = this.getAddFunction();
+          }
+        };
+      } else {
+        return (value: T) => {
+          this.buffer.unshift(value);
+          if (this.buffer.length >= this.capacity) {
+            this.add = this.getAddFunction();
+          }
+        };
+      }
+    }
+
+    if (this.tail) {
+      return (value: T) => {
+        this.buffer[this.index] = value;
+        this.index = (this.index + 1) % this.buffer.length;
+      };
+    }
+
+    // Append values to the head
+    return (value: T) => {
+      let idx = this.index - 1;
+      if (idx < 0) {
+        idx = this.buffer.length - 1;
+      }
+      this.buffer[idx] = value;
+      this.index = idx;
+    };
+  }
+
+  setCapacity(v: number) {
+    if (this.capacity === v) {
+      return;
+    }
+    // Make a copy so it is in order and new additions can be at the head or tail
+    const copy = this.toArray();
+    if (v > this.length) {
+      this.buffer = copy;
+    } else if (v < this.capacity) {
+      // Shrink the buffer
+      const delta = this.length - v;
+      if (this.tail) {
+        this.buffer = copy.slice(delta, copy.length); // Keep last items
+      } else {
+        this.buffer = copy.slice(0, copy.length - delta); // Keep first items
+      }
+    }
+    this.capacity = v;
+    this.index = 0;
+    this.add = this.getAddFunction();
+  }
+
+  setAppendMode(mode: 'head' | 'tail') {
+    const tail = 'head' !== mode;
+    if (tail !== this.tail) {
+      this.buffer = this.toArray().reverse();
+      this.index = 0;
+      this.tail = tail;
+      this.add = this.getAddFunction();
+    }
+  }
+
+  reverse() {
+    this.buffer.reverse();
+  }
+
+  /**
+   * Add the value to the buffer
+   */
+  add: (value: T) => void;
+
+  get(index: number) {
+    return this.buffer[(index + this.index) % this.buffer.length];
+  }
+
+  set(index: number, value: T) {
+    this.buffer[(index + this.index) % this.buffer.length] = value;
+  }
+
+  get length() {
+    return this.buffer.length;
+  }
+
+  toArray(): T[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): T[] {
+    return vectorToArray(this);
+  }
+}

packages/grafana-data/src/vector/ConstantVector.test.ts

@@ -0,0 +1,17 @@
+import { ConstantVector } from './ConstantVector';
+
+describe('ConstantVector', () => {
+  it('should support constant values', () => {
+    const value = 3.5;
+    const v = new ConstantVector(value, 7);
+    expect(v.length).toEqual(7);
+
+    expect(v.get(0)).toEqual(value);
+    expect(v.get(1)).toEqual(value);
+
+    // Now check all of them
+    for (let i = 0; i < 10; i++) {
+      expect(v.get(i)).toEqual(value);
+    }
+  });
+});

packages/grafana-data/src/vector/ConstantVector.ts

@@ -0,0 +1,22 @@
+import { Vector } from '../types/vector';
+
+export class ConstantVector<T = any> implements Vector<T> {
+  constructor(private value: T, private len: number) {}
+
+  get length() {
+    return this.len;
+  }
+
+  get(index: number): T {
+    return this.value;
+  }
+
+  toArray(): T[] {
+    const arr = new Array<T>(this.length);
+    return arr.fill(this.value);
+  }
+
+  toJSON(): T[] {
+    return this.toArray();
+  }
+}

packages/grafana-data/src/vector/ScaledVector.test.ts

@@ -0,0 +1,15 @@
+import { ArrayVector } from './ArrayVector';
+import { ScaledVector } from './ScaledVector';
+
+describe('ScaledVector', () => {
+  it('should support multiply operations', () => {
+    const source = new ArrayVector([1, 2, 3, 4]);
+    const scale = 2.456;
+    const v = new ScaledVector(source, scale);
+    expect(v.length).toEqual(source.length);
+    //  expect(v.push(10)).toEqual(source.length); // not implemented
+    for (let i = 0; i < 10; i++) {
+      expect(v.get(i)).toEqual(source.get(i) * scale);
+    }
+  });
+});

packages/grafana-data/src/vector/ScaledVector.ts

@@ -0,0 +1,22 @@
+import { Vector } from '../types/vector';
+import { vectorToArray } from './vectorToArray';
+
+export class ScaledVector implements Vector<number> {
+  constructor(private source: Vector<number>, private scale: number) {}
+
+  get length(): number {
+    return this.source.length;
+  }
+
+  get(index: number): number {
+    return this.source.get(index) * this.scale;
+  }
+
+  toArray(): number[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): number[] {
+    return vectorToArray(this);
+  }
+}

packages/grafana-data/src/vector/SortedVector.ts

@@ -0,0 +1,25 @@
+import { Vector } from '../types/vector';
+import { vectorToArray } from './vectorToArray';
+
+/**
+ * Values are returned in the order defined by the input parameter
+ */
+export class SortedVector<T = any> implements Vector<T> {
+  constructor(private source: Vector<T>, private order: number[]) {}
+
+  get length(): number {
+    return this.source.length;
+  }
+
+  get(index: number): T {
+    return this.source.get(this.order[index]);
+  }
+
+  toArray(): T[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): T[] {
+    return vectorToArray(this);
+  }
+}

packages/grafana-data/src/vector/index.ts

@@ -0,0 +1,6 @@
+export * from './AppendedVectors';
+export * from './ArrayVector';
+export * from './CircularVector';
+export * from './ConstantVector';
+export * from './ScaledVector';
+export * from './SortedVector';

packages/grafana-data/src/vector/vectorToArray.ts

@@ -0,0 +1,9 @@
+import { Vector } from '../types/vector';
+
+export function vectorToArray<T>(v: Vector<T>): T[] {
+  const arr: T[] = [];
+  for (let i = 0; i < v.length; i++) {
+    arr[i] = v.get(i);
+  }
+  return arr;
+}