grafana/data: Reorganise code (#19136)

* Organise data frame and vectors code

* Organise transformations

* Move dataframe utils to dataframe dir

* Organise datetime utils

* Organise text utils

* Organise logs utils

* Revert "Organise logs utils"

This reverts commit c24115c755.

* registry -> Registry

* Transformations reorg
This commit is contained in:
Dominik Prokop
2019-09-17 08:23:47 +02:00
committed by GitHub
parent 330cd597ec
commit 5c0f424d1b
76 changed files with 813 additions and 768 deletions
@@ -0,0 +1,138 @@
import sinon, { SinonFakeTimers } from 'sinon';
import each from 'lodash/each';
import * as dateMath from './datemath';
import { dateTime, DurationUnit, DateTime } from './moment_wrapper';
describe('DateMath', () => {
const spans: DurationUnit[] = ['s', 'm', 'h', 'd', 'w', 'M', 'y'];
const anchor = '2014-01-01T06:06:06.666Z';
const unix = dateTime(anchor).valueOf();
const format = 'YYYY-MM-DDTHH:mm:ss.SSSZ';
let clock: SinonFakeTimers;
describe('errors', () => {
it('should return undefined if passed empty string', () => {
expect(dateMath.parse('')).toBe(undefined);
});
it('should return undefined if I pass an operator besides [+-/]', () => {
expect(dateMath.parse('now&1d')).toBe(undefined);
});
it('should return undefined if I pass a unit besides' + spans.toString(), () => {
expect(dateMath.parse('now+5f')).toBe(undefined);
});
it('should return undefined if rounding unit is not 1', () => {
expect(dateMath.parse('now/2y')).toBe(undefined);
expect(dateMath.parse('now/0.5y')).toBe(undefined);
});
it('should not go into an infinite loop when missing a unit', () => {
expect(dateMath.parse('now-0')).toBe(undefined);
expect(dateMath.parse('now-00')).toBe(undefined);
});
});
it('now/d should set to start of current day', () => {
const expected = new Date();
expected.setHours(0);
expected.setMinutes(0);
expected.setSeconds(0);
expected.setMilliseconds(0);
const startOfDay = dateMath.parse('now/d', false)!.valueOf();
expect(startOfDay).toBe(expected.getTime());
});
it('now/d on a utc dashboard should be start of the current day in UTC time', () => {
const today = new Date();
const expected = new Date(Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate(), 0, 0, 0, 0));
const startOfDay = dateMath.parse('now/d', false, 'utc')!.valueOf();
expect(startOfDay).toBe(expected.getTime());
});
describe('subtraction', () => {
let now: DateTime;
let anchored: DateTime;
beforeEach(() => {
clock = sinon.useFakeTimers(unix);
now = dateTime();
anchored = dateTime(anchor);
});
each(spans, span => {
const nowEx = 'now-5' + span;
const thenEx = anchor + '||-5' + span;
it('should return 5' + span + ' ago', () => {
expect(dateMath.parse(nowEx)!.format(format)).toEqual(now.subtract(5, span).format(format));
});
it('should return 5' + span + ' before ' + anchor, () => {
expect(dateMath.parse(thenEx)!.format(format)).toEqual(anchored.subtract(5, span).format(format));
});
});
afterEach(() => {
clock.restore();
});
});
describe('rounding', () => {
let now: DateTime;
beforeEach(() => {
clock = sinon.useFakeTimers(unix);
now = dateTime();
});
each(spans, span => {
it('should round now to the beginning of the ' + span, () => {
expect(dateMath.parse('now/' + span)!.format(format)).toEqual(now.startOf(span).format(format));
});
it('should round now to the end of the ' + span, () => {
expect(dateMath.parse('now/' + span, true)!.format(format)).toEqual(now.endOf(span).format(format));
});
});
afterEach(() => {
clock.restore();
});
});
describe('isValid', () => {
it('should return false when invalid date text', () => {
expect(dateMath.isValid('asd')).toBe(false);
});
it('should return true when valid date text', () => {
expect(dateMath.isValid('now-1h')).toBe(true);
});
});
describe('relative time to date parsing', () => {
it('should handle negative time', () => {
const date = dateMath.parseDateMath('-2d', dateTime([2014, 1, 5]));
expect(date!.valueOf()).toEqual(dateTime([2014, 1, 3]).valueOf());
});
it('should handle multiple math expressions', () => {
const date = dateMath.parseDateMath('-2d-6h', dateTime([2014, 1, 5]));
expect(date!.valueOf()).toEqual(dateTime([2014, 1, 2, 18]).valueOf());
});
it('should return false when invalid expression', () => {
const date = dateMath.parseDateMath('2', dateTime([2014, 1, 5]));
expect(date).toEqual(undefined);
});
it('should strip whitespace from string', () => {
const date = dateMath.parseDateMath(' - 2d', dateTime([2014, 1, 5]));
expect(date!.valueOf()).toEqual(dateTime([2014, 1, 3]).valueOf());
});
});
});
@@ -0,0 +1,158 @@
import includes from 'lodash/includes';
import isDate from 'lodash/isDate';
import { DateTime, dateTime, dateTimeForTimeZone, ISO_8601, isDateTime, DurationUnit } from './moment_wrapper';
import { TimeZone } from '../types/index';
const units: DurationUnit[] = ['y', 'M', 'w', 'd', 'h', 'm', 's'];
export function isMathString(text: string | DateTime | Date): boolean {
if (!text) {
return false;
}
if (typeof text === 'string' && (text.substring(0, 3) === 'now' || text.includes('||'))) {
return true;
} else {
return false;
}
}
/**
* Parses different types input to a moment instance. There is a specific formatting language that can be used
* if text arg is string. See unit tests for examples.
* @param text
* @param roundUp See parseDateMath function.
* @param timezone Only string 'utc' is acceptable here, for anything else, local timezone is used.
*/
export function parse(text: string | DateTime | Date, roundUp?: boolean, timezone?: TimeZone): DateTime | undefined {
if (!text) {
return undefined;
}
if (typeof text !== 'string') {
if (isDateTime(text)) {
return text;
}
if (isDate(text)) {
return dateTime(text);
}
// We got some non string which is not a moment nor Date. TS should be able to check for that but not always.
return undefined;
} else {
let time;
let mathString = '';
let index;
let parseString;
if (text.substring(0, 3) === 'now') {
time = dateTimeForTimeZone(timezone);
mathString = text.substring('now'.length);
} else {
index = text.indexOf('||');
if (index === -1) {
parseString = text;
mathString = ''; // nothing else
} else {
parseString = text.substring(0, index);
mathString = text.substring(index + 2);
}
// We're going to just require ISO8601 timestamps, k?
time = dateTime(parseString, ISO_8601);
}
if (!mathString.length) {
return time;
}
return parseDateMath(mathString, time, roundUp);
}
}
/**
* Checks if text is a valid date which in this context means that it is either a Moment instance or it can be parsed
* by parse function. See parse function to see what is considered acceptable.
* @param text
*/
export function isValid(text: string | DateTime): boolean {
const date = parse(text);
if (!date) {
return false;
}
if (isDateTime(date)) {
return date.isValid();
}
return false;
}
/**
* Parses math part of the time string and shifts supplied time according to that math. See unit tests for examples.
* @param mathString
* @param time
* @param roundUp If true it will round the time to endOf time unit, otherwise to startOf time unit.
*/
// TODO: Had to revert Andrejs `time: moment.Moment` to `time: any`
export function parseDateMath(mathString: string, time: any, roundUp?: boolean): DateTime | undefined {
const strippedMathString = mathString.replace(/\s/g, '');
const dateTime = time;
let i = 0;
const len = strippedMathString.length;
while (i < len) {
const c = strippedMathString.charAt(i++);
let type;
let num;
let unit;
if (c === '/') {
type = 0;
} else if (c === '+') {
type = 1;
} else if (c === '-') {
type = 2;
} else {
return undefined;
}
if (isNaN(parseInt(strippedMathString.charAt(i), 10))) {
num = 1;
} else if (strippedMathString.length === 2) {
num = strippedMathString.charAt(i);
} else {
const numFrom = i;
while (!isNaN(parseInt(strippedMathString.charAt(i), 10))) {
i++;
if (i > 10) {
return undefined;
}
}
num = parseInt(strippedMathString.substring(numFrom, i), 10);
}
if (type === 0) {
// rounding is only allowed on whole, single, units (eg M or 1M, not 0.5M or 2M)
if (num !== 1) {
return undefined;
}
}
unit = strippedMathString.charAt(i++);
if (!includes(units, unit)) {
return undefined;
} else {
if (type === 0) {
if (roundUp) {
dateTime.endOf(unit);
} else {
dateTime.startOf(unit);
}
} else if (type === 1) {
dateTime.add(num, unit);
} else if (type === 2) {
dateTime.subtract(num, unit);
}
}
}
return dateTime;
}
@@ -0,0 +1,5 @@
// Names are too general to export globally
import * as dateMath from './datemath';
import * as rangeUtil from './rangeutil';
export * from './moment_wrapper';
export { dateMath, rangeUtil };
@@ -0,0 +1,115 @@
import { TimeZone } from '../types/time';
/* tslint:disable:import-blacklist ban ban-types */
import moment, { Moment, MomentInput, DurationInputArg1 } from 'moment';
export interface DateTimeBuiltinFormat {
__momentBuiltinFormatBrand: any;
}
export const ISO_8601: DateTimeBuiltinFormat = moment.ISO_8601;
export type DateTimeInput = Date | string | number | Array<string | number> | DateTime; // null | undefined
export type FormatInput = string | DateTimeBuiltinFormat | undefined;
export type DurationInput = string | number | DateTimeDuration;
export type DurationUnit =
| 'year'
| 'years'
| 'y'
| 'month'
| 'months'
| 'M'
| 'week'
| 'weeks'
| 'w'
| 'day'
| 'days'
| 'd'
| 'hour'
| 'hours'
| 'h'
| 'minute'
| 'minutes'
| 'm'
| 'second'
| 'seconds'
| 's'
| 'millisecond'
| 'milliseconds'
| 'ms'
| 'quarter'
| 'quarters'
| 'Q';
export interface DateTimeLocale {
firstDayOfWeek: () => number;
}
export interface DateTimeDuration {
asHours: () => number;
hours: () => number;
minutes: () => number;
seconds: () => number;
asSeconds: () => number;
}
export interface DateTime extends Object {
add: (amount?: DateTimeInput, unit?: DurationUnit) => DateTime;
set: (unit: DurationUnit, amount: DateTimeInput) => void;
diff: (amount: DateTimeInput, unit?: DurationUnit, truncate?: boolean) => number;
endOf: (unitOfTime: DurationUnit) => DateTime;
format: (formatInput?: FormatInput) => string;
fromNow: (withoutSuffix?: boolean) => string;
from: (formaInput: DateTimeInput) => string;
isSame: (input?: DateTimeInput, granularity?: DurationUnit) => boolean;
isValid: () => boolean;
local: () => DateTime;
locale: (locale: string) => DateTime;
startOf: (unitOfTime: DurationUnit) => DateTime;
subtract: (amount?: DateTimeInput, unit?: DurationUnit) => DateTime;
toDate: () => Date;
toISOString: () => string;
isoWeekday: (day?: number | string) => number | string;
valueOf: () => number;
unix: () => number;
utc: () => DateTime;
utcOffset: () => number;
hour?: () => number;
minute?: () => number;
}
export const setLocale = (language: string) => {
moment.locale(language);
};
export const getLocaleData = (): DateTimeLocale => {
return moment.localeData();
};
export const isDateTime = (value: any): value is DateTime => {
return moment.isMoment(value);
};
export const toUtc = (input?: DateTimeInput, formatInput?: FormatInput): DateTime => {
return moment.utc(input as MomentInput, formatInput) as DateTime;
};
export const toDuration = (input?: DurationInput, unit?: DurationUnit): DateTimeDuration => {
return moment.duration(input as DurationInputArg1, unit) as DateTimeDuration;
};
export const dateTime = (input?: DateTimeInput, formatInput?: FormatInput): DateTime => {
return moment(input as MomentInput, formatInput) as DateTime;
};
export const dateTimeAsMoment = (input?: DateTimeInput) => {
return dateTime(input) as Moment;
};
export const dateTimeForTimeZone = (
timezone?: TimeZone,
input?: DateTimeInput,
formatInput?: FormatInput
): DateTime => {
if (timezone === 'utc') {
return toUtc(input, formatInput);
}
return dateTime(input, formatInput);
};
@@ -0,0 +1,170 @@
import each from 'lodash/each';
import groupBy from 'lodash/groupBy';
import { RawTimeRange } from '../types/time';
import * as dateMath from './datemath';
import { isDateTime, DateTime } from './moment_wrapper';
const spans: { [key: string]: { display: string; section?: number } } = {
s: { display: 'second' },
m: { display: 'minute' },
h: { display: 'hour' },
d: { display: 'day' },
w: { display: 'week' },
M: { display: 'month' },
y: { display: 'year' },
};
const rangeOptions = [
{ from: 'now/d', to: 'now/d', display: 'Today', section: 2 },
{ from: 'now/d', to: 'now', display: 'Today so far', section: 2 },
{ from: 'now/w', to: 'now/w', display: 'This week', section: 2 },
{ from: 'now/w', to: 'now', display: 'This week so far', section: 2 },
{ from: 'now/M', to: 'now/M', display: 'This month', section: 2 },
{ from: 'now/M', to: 'now', display: 'This month so far', section: 2 },
{ from: 'now/y', to: 'now/y', display: 'This year', section: 2 },
{ from: 'now/y', to: 'now', display: 'This year so far', section: 2 },
{ from: 'now-1d/d', to: 'now-1d/d', display: 'Yesterday', section: 1 },
{
from: 'now-2d/d',
to: 'now-2d/d',
display: 'Day before yesterday',
section: 1,
},
{
from: 'now-7d/d',
to: 'now-7d/d',
display: 'This day last week',
section: 1,
},
{ from: 'now-1w/w', to: 'now-1w/w', display: 'Previous week', section: 1 },
{ from: 'now-1M/M', to: 'now-1M/M', display: 'Previous month', section: 1 },
{ from: 'now-1y/y', to: 'now-1y/y', display: 'Previous year', section: 1 },
{ from: 'now-5m', to: 'now', display: 'Last 5 minutes', section: 3 },
{ from: 'now-15m', to: 'now', display: 'Last 15 minutes', section: 3 },
{ from: 'now-30m', to: 'now', display: 'Last 30 minutes', section: 3 },
{ from: 'now-1h', to: 'now', display: 'Last 1 hour', section: 3 },
{ from: 'now-3h', to: 'now', display: 'Last 3 hours', section: 3 },
{ from: 'now-6h', to: 'now', display: 'Last 6 hours', section: 3 },
{ from: 'now-12h', to: 'now', display: 'Last 12 hours', section: 3 },
{ from: 'now-24h', to: 'now', display: 'Last 24 hours', section: 3 },
{ from: 'now-2d', to: 'now', display: 'Last 2 days', section: 0 },
{ from: 'now-7d', to: 'now', display: 'Last 7 days', section: 0 },
{ from: 'now-30d', to: 'now', display: 'Last 30 days', section: 0 },
{ from: 'now-90d', to: 'now', display: 'Last 90 days', section: 0 },
{ from: 'now-6M', to: 'now', display: 'Last 6 months', section: 0 },
{ from: 'now-1y', to: 'now', display: 'Last 1 year', section: 0 },
{ from: 'now-2y', to: 'now', display: 'Last 2 years', section: 0 },
{ from: 'now-5y', to: 'now', display: 'Last 5 years', section: 0 },
];
const absoluteFormat = 'YYYY-MM-DD HH:mm:ss';
const rangeIndex: any = {};
each(rangeOptions, (frame: any) => {
rangeIndex[frame.from + ' to ' + frame.to] = frame;
});
export function getRelativeTimesList(timepickerSettings: any, currentDisplay: any) {
const groups = groupBy(rangeOptions, (option: any) => {
option.active = option.display === currentDisplay;
return option.section;
});
// _.each(timepickerSettings.time_options, (duration: string) => {
// let info = describeTextRange(duration);
// if (info.section) {
// groups[info.section].push(info);
// }
// });
return groups;
}
function formatDate(date: DateTime) {
return date.format(absoluteFormat);
}
// handles expressions like
// 5m
// 5m to now/d
// now/d to now
// now/d
// if no to <expr> then to now is assumed
export function describeTextRange(expr: any) {
const isLast = expr.indexOf('+') !== 0;
if (expr.indexOf('now') === -1) {
expr = (isLast ? 'now-' : 'now') + expr;
}
let opt = rangeIndex[expr + ' to now'];
if (opt) {
return opt;
}
if (isLast) {
opt = { from: expr, to: 'now' };
} else {
opt = { from: 'now', to: expr };
}
const parts = /^now([-+])(\d+)(\w)/.exec(expr);
if (parts) {
const unit = parts[3];
const amount = parseInt(parts[2], 10);
const span = spans[unit];
if (span) {
opt.display = isLast ? 'Last ' : 'Next ';
opt.display += amount + ' ' + span.display;
opt.section = span.section;
if (amount > 1) {
opt.display += 's';
}
}
} else {
opt.display = opt.from + ' to ' + opt.to;
opt.invalid = true;
}
return opt;
}
export function describeTimeRange(range: RawTimeRange): string {
const option = rangeIndex[range.from.toString() + ' to ' + range.to.toString()];
if (option) {
return option.display;
}
if (isDateTime(range.from) && isDateTime(range.to)) {
return formatDate(range.from) + ' to ' + formatDate(range.to);
}
if (isDateTime(range.from)) {
const toMoment = dateMath.parse(range.to, true);
return toMoment ? formatDate(range.from) + ' to ' + toMoment.fromNow() : '';
}
if (isDateTime(range.to)) {
const from = dateMath.parse(range.from, false);
return from ? from.fromNow() + ' to ' + formatDate(range.to) : '';
}
if (range.to.toString() === 'now') {
const res = describeTextRange(range.from);
return res.display;
}
return range.from.toString() + ' to ' + range.to.toString();
}
export const isValidTimeSpan = (value: string) => {
if (value.indexOf('$') === 0 || value.indexOf('+$') === 0) {
return true;
}
const info = describeTextRange(value);
return info.invalid !== true;
};