From 583334df051ca736140663256c6bed2303c34f84 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 2 Nov 2018 08:25:36 +0100 Subject: [PATCH 1/7] Explore: Logging graph overview and view options - Logging gets a graph for log distribution (currently per stream, but I think I'll change that to per log-level) - added grid columns for timestamp and unique labels - show common labels of streams - View options to show/hide time columns, label columns - created `--small` modifier for Switch CSS classes - merging of streams is now a datasource responsibility --- public/app/core/components/Switch/Switch.tsx | 11 +- public/app/core/logs_model.ts | 34 +++-- public/app/features/explore/Explore.tsx | 16 ++- public/app/features/explore/Graph.tsx | 5 +- public/app/features/explore/Logs.tsx | 109 ++++++++++++++- .../plugins/datasource/logging/datasource.ts | 13 +- .../logging/result_transformer.test.ts | 56 +++++++- .../datasource/logging/result_transformer.ts | 132 +++++++++++++++++- public/sass/components/_gf-form.scss | 5 + public/sass/components/_switch.scss | 15 +- public/sass/pages/_explore.scss | 37 ++++- 11 files changed, 397 insertions(+), 36 deletions(-) diff --git a/public/app/core/components/Switch/Switch.tsx b/public/app/core/components/Switch/Switch.tsx index ba09267ebd2..a4bb73a291b 100644 --- a/public/app/core/components/Switch/Switch.tsx +++ b/public/app/core/components/Switch/Switch.tsx @@ -5,6 +5,7 @@ export interface Props { label: string; checked: boolean; labelClass?: string; + small?: boolean; switchClass?: string; onChange: (event) => any; } @@ -24,10 +25,14 @@ export class Switch extends PureComponent { }; render() { - const { labelClass, switchClass, label, checked } = this.props; + const { labelClass = '', switchClass = '', label, checked, small } = this.props; const labelId = `check-${this.state.id}`; - const labelClassName = `gf-form-label ${labelClass} pointer`; - const switchClassName = `gf-form-switch ${switchClass}`; + let labelClassName = `gf-form-label ${labelClass} pointer`; + let switchClassName = `gf-form-switch ${switchClass}`; + if (small) { + labelClassName += ' gf-form-label--small'; + switchClassName += ' gf-form-switch--small'; + } return (
diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index e6f317dbeb7..ca7899db7d8 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { TimeSeries } from 'app/core/core'; export enum LogLevel { crit = 'crit', @@ -19,25 +20,34 @@ export interface LogSearchMatch { export interface LogRow { key: string; entry: string; + labels: string; logLevel: LogLevel; timestamp: string; timeFromNow: string; + timeJs: number; timeLocal: string; searchWords?: string[]; } -export interface LogsModel { - rows: LogRow[]; +export interface LogsMetaItem { + label: string; + value: string; } -export function mergeStreams(streams: LogsModel[], limit?: number): LogsModel { - const combinedEntries = streams.reduce((acc, stream) => { - return [...acc, ...stream.rows]; - }, []); - const sortedEntries = _.chain(combinedEntries) - .sortBy('timestamp') - .reverse() - .slice(0, limit || combinedEntries.length) - .value(); - return { rows: sortedEntries }; +export interface LogsModel { + meta?: LogsMetaItem[]; + rows: LogRow[]; + series?: TimeSeries[]; +} + +export interface LogsStream { + labels: string; + entries: LogsStreamEntry[]; + parsedLabels: { [key: string]: string }; + graphSeries: TimeSeries; +} + +export interface LogsStreamEntry { + line: string; + timestamp: string; } diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index af771bad5dd..6ec6c79ac5f 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -25,7 +25,6 @@ import ErrorBoundary from './ErrorBoundary'; import TimePicker from './TimePicker'; import { ensureQueries, generateQueryKey, hasQuery } from './utils/query'; import { DataSource } from 'app/types/datasources'; -import { mergeStreams } from 'app/core/logs_model'; const MAX_HISTORY_ITEMS = 100; @@ -770,9 +769,14 @@ export class Explore extends React.PureComponent { new TableModel(), ...queryTransactions.filter(qt => qt.resultType === 'Table' && qt.done && qt.result).map(qt => qt.result) ); - const logsResult = mergeStreams( - queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result) - ); + const logsResult = + datasource && datasource.mergeStreams + ? datasource.mergeStreams( + _.flatten( + queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result) + ) + ) + : undefined; const loading = queryTransactions.some(qt => !qt.done); const showStartPages = StartPage && queryTransactions.length === 0; const viewModeCount = [supportsGraph, supportsLogs, supportsTable].filter(m => m).length; @@ -903,7 +907,9 @@ export class Explore extends React.PureComponent { ) : null} - {supportsLogs && showingLogs ? : null} + {supportsLogs && showingLogs ? ( + + ) : null} )} diff --git a/public/app/features/explore/Graph.tsx b/public/app/features/explore/Graph.tsx index bbb055067a2..6c22ca67509 100644 --- a/public/app/features/explore/Graph.tsx +++ b/public/app/features/explore/Graph.tsx @@ -79,6 +79,7 @@ interface GraphProps { range: RawTimeRange; split?: boolean; size?: { width: number; height: number }; + userOptions?: any; } interface GraphState { @@ -122,7 +123,7 @@ export class Graph extends PureComponent { }; draw() { - const { range, size } = this.props; + const { range, size, userOptions = {} } = this.props; const data = this.getGraphData(); const $el = $(`#${this.props.id}`); @@ -153,12 +154,14 @@ export class Graph extends PureComponent { max: max, label: 'Datetime', ticks: ticks, + timezone: 'browser', timeformat: time_format(ticks, min, max), }, }; const options = { ...FLOT_OPTIONS, ...dynamicOptions, + ...userOptions, }; $.plot($el, series, options); } diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 278c5ee016d..5630d1de8f6 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -1,29 +1,130 @@ import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; +import { RawTimeRange } from 'app/types/series'; import { LogsModel } from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; +import { Switch } from 'app/core/components/Switch/Switch'; + +import Graph from './Graph'; + +const graphOptions = { + series: { + bars: { + show: true, + }, + }, + yaxis: { + tickDecimals: 0, + }, +}; interface LogsProps { className?: string; data: LogsModel; loading: boolean; + position: string; + range?: RawTimeRange; } -export default class Logs extends PureComponent { +interface LogsState { + showLabels: boolean; + showLocalTime: boolean; + showUtc: boolean; +} + +export default class Logs extends PureComponent { + state = { + showLabels: true, + showLocalTime: true, + showUtc: false, + }; + + onChangeLabels = (event: React.SyntheticEvent) => { + const target = event.target as HTMLInputElement; + this.setState({ + showLabels: target.checked, + }); + }; + + onChangeLocalTime = (event: React.SyntheticEvent) => { + const target = event.target as HTMLInputElement; + this.setState({ + showLocalTime: target.checked, + }); + }; + + onChangeUtc = (event: React.SyntheticEvent) => { + const target = event.target as HTMLInputElement; + this.setState({ + showUtc: target.checked, + }); + }; + render() { - const { className = '', data, loading = false } = this.props; + const { className = '', data, loading = false, position, range } = this.props; + const { showLabels, showLocalTime, showUtc } = this.state; const hasData = data && data.rows && data.rows.length > 0; + const cssColumnSizes = ['4px']; + if (showUtc) { + cssColumnSizes.push('minmax(100px, max-content)'); + } + if (showLocalTime) { + cssColumnSizes.push('minmax(100px, max-content)'); + } + if (showLabels) { + cssColumnSizes.push('minmax(100px, 25%)'); + } + cssColumnSizes.push('1fr'); + const logEntriesStyle = { + gridTemplateColumns: cssColumnSizes.join(' '), + }; + return (
+
+ +
+ +
+
+ + + + {hasData && + data.meta && ( +
+ {data.meta.map(item => ( +
+ {item.label}: + {item.value} +
+ ))} +
+ )} +
+
+
{loading &&
} -
+
{hasData && data.rows.map(row => (
-
{row.timeLocal}
+ {showUtc &&
{row.timestamp}
} + {showLocalTime &&
{row.timeLocal}
} + {showLabels && ( +
+ {row.labels} +
+ )}
processStream(stream, DEFAULT_LIMIT)); + return { data: processedStreams }; }); } diff --git a/public/app/plugins/datasource/logging/result_transformer.test.ts b/public/app/plugins/datasource/logging/result_transformer.test.ts index c1e6913a388..28debe41585 100644 --- a/public/app/plugins/datasource/logging/result_transformer.test.ts +++ b/public/app/plugins/datasource/logging/result_transformer.test.ts @@ -1,6 +1,6 @@ import { LogLevel } from 'app/core/logs_model'; -import { getLogLevel } from './result_transformer'; +import { findCommonLabels, findUncommonLabels, formatLabels, getLogLevel, parseLabels } from './result_transformer'; describe('getLoglevel()', () => { it('returns no log level on empty line', () => { @@ -20,3 +20,57 @@ describe('getLoglevel()', () => { expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn); }); }); + +describe('parseLabels()', () => { + it('returns no labels on emtpy labels string', () => { + expect(parseLabels('')).toEqual({}); + expect(parseLabels('{}')).toEqual({}); + }); + + it('returns labels on labels string', () => { + expect(parseLabels('{foo="bar", baz="42"}')).toEqual({ foo: '"bar"', baz: '"42"' }); + }); +}); + +describe('formatLabels()', () => { + it('returns no labels on emtpy label set', () => { + expect(formatLabels({})).toEqual(''); + expect(formatLabels({}, 'foo')).toEqual('foo'); + }); + + it('returns label string on label set', () => { + expect(formatLabels({ foo: '"bar"', baz: '"42"' })).toEqual('{baz="42", foo="bar"}'); + }); +}); + +describe('findCommonLabels()', () => { + it('returns no common labels on empty sets', () => { + expect(findCommonLabels([{}])).toEqual({}); + expect(findCommonLabels([{}, {}])).toEqual({}); + }); + + it('returns no common labels on differing sets', () => { + expect(findCommonLabels([{ foo: '"bar"' }, {}])).toEqual({}); + expect(findCommonLabels([{}, { foo: '"bar"' }])).toEqual({}); + expect(findCommonLabels([{ baz: '42' }, { foo: '"bar"' }])).toEqual({}); + expect(findCommonLabels([{ foo: '42', baz: '"bar"' }, { foo: '"bar"' }])).toEqual({}); + }); + + it('returns the single labels set as common labels', () => { + expect(findCommonLabels([{ foo: '"bar"' }])).toEqual({ foo: '"bar"' }); + }); +}); + +describe('findUncommonLabels()', () => { + it('returns no uncommon labels on empty sets', () => { + expect(findUncommonLabels({}, {})).toEqual({}); + }); + + it('returns all labels given no common labels', () => { + expect(findUncommonLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' }); + }); + + it('returns all labels except the common labels', () => { + expect(findUncommonLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' }); + }); +}); diff --git a/public/app/plugins/datasource/logging/result_transformer.ts b/public/app/plugins/datasource/logging/result_transformer.ts index 526a9c7da2c..8aa7ebc12e0 100644 --- a/public/app/plugins/datasource/logging/result_transformer.ts +++ b/public/app/plugins/datasource/logging/result_transformer.ts @@ -1,7 +1,9 @@ import _ from 'lodash'; import moment from 'moment'; -import { LogLevel, LogsModel, LogRow } from 'app/core/logs_model'; +import { LogLevel, LogsMetaItem, LogsModel, LogRow, LogsStream } from 'app/core/logs_model'; +import { TimeSeries } from 'app/core/core'; +import colors from 'app/core/utils/colors'; export function getLogLevel(line: string): LogLevel { if (!line) { @@ -19,11 +21,65 @@ export function getLogLevel(line: string): LogLevel { return level; } +const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g; +export function parseLabels(labels: string): { [key: string]: string } { + const labelsByKey = {}; + labels.replace(labelRegexp, (_, key, operator, value) => { + labelsByKey[key] = value; + return ''; + }); + return labelsByKey; +} + +export function findCommonLabels(labelsSets: any[]) { + return labelsSets.reduce((acc, labels) => { + if (!labels) { + throw new Error('Need parsed labels to find common labels.'); + } + if (!acc) { + // Initial set + acc = { ...labels }; + } else { + // Remove incoming labels that are missing or not matching in value + Object.keys(labels).forEach(key => { + if (acc[key] === undefined || acc[key] !== labels[key]) { + delete acc[key]; + } + }); + // Remove common labels that are missing from incoming label set + Object.keys(acc).forEach(key => { + if (labels[key] === undefined) { + delete acc[key]; + } + }); + } + return acc; + }, undefined); +} + +export function findUncommonLabels(labels, commonLabels) { + const uncommonLabels = { ...labels }; + Object.keys(commonLabels).forEach(key => { + delete uncommonLabels[key]; + }); + return uncommonLabels; +} + +export function formatLabels(labels, defaultValue = '') { + if (!labels || Object.keys(labels).length === 0) { + return defaultValue; + } + const labelKeys = Object.keys(labels).sort(); + const cleanSelector = labelKeys.map(key => `${key}=${labels[key]}`).join(', '); + return ['{', cleanSelector, '}'].join(''); +} + export function processEntry(entry: { line: string; timestamp: string }, stream): LogRow { const { line, timestamp } = entry; const { labels } = stream; const key = `EK${timestamp}${labels}`; const time = moment(timestamp); + const timeJs = time.valueOf(); const timeFromNow = time.fromNow(); const timeLocal = time.format('YYYY-MM-DD HH:mm:ss'); const logLevel = getLogLevel(line); @@ -32,21 +88,89 @@ export function processEntry(entry: { line: string; timestamp: string }, stream) key, logLevel, timeFromNow, + timeJs, timeLocal, entry: line, + labels: formatLabels(labels), searchWords: [stream.search], timestamp: timestamp, }; } -export function processStreams(streams, limit?: number): LogsModel { +export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel { + // Find meta data + const commonLabels = findCommonLabels(streams.map(stream => stream.parsedLabels)); + const meta: LogsMetaItem[] = [ + { + label: 'Common labels', + value: formatLabels(commonLabels), + }, + ]; + + // Flatten entries of streams const combinedEntries = streams.reduce((acc, stream) => { - return [...acc, ...stream.entries.map(entry => processEntry(entry, stream))]; + // Overwrite labels to be only the non-common ones + const labels = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels)); + return [ + ...acc, + ...stream.entries.map(entry => ({ + ...entry, + labels, + })), + ]; }, []); + + const commonLabelsAlias = + streams.length === 1 ? formatLabels(commonLabels) : `Stream with common labels ${formatLabels(commonLabels)}`; + const series = streams.map((stream, index) => { + const colorIndex = index % colors.length; + stream.graphSeries.setColor(colors[colorIndex]); + stream.graphSeries.alias = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels), commonLabelsAlias); + return stream.graphSeries; + }); + const sortedEntries = _.chain(combinedEntries) .sortBy('timestamp') .reverse() .slice(0, limit || combinedEntries.length) .value(); - return { rows: sortedEntries }; + + meta.push({ + label: 'Limit', + value: `${limit} (${sortedEntries.length} returned)`, + }); + + return { meta, series, rows: sortedEntries }; +} + +export function processStream(stream: LogsStream, limit?: number): LogsStream { + const sortedEntries: any[] = _.chain(stream.entries) + .map(entry => processEntry(entry, stream)) + .sortBy('timestamp') + .reverse() + .slice(0, limit || stream.entries.length) + .value(); + + // Build graph data + let previousTime; + const datapoints = sortedEntries.reduce((acc, entry, index) => { + // Bucket to nearest minute + const time = Math.round(entry.timeJs / 1000 / 60) * 1000 * 60; + // Entry for time + if (time === previousTime) { + acc[acc.length - 1][0]++; + } else { + acc.push([1, time]); + previousTime = time; + } + return acc; + }, []); + const graphSeries = new TimeSeries({ datapoints, alias: stream.labels }); + + return { + ...stream, + graphSeries, + entries: sortedEntries, + parsedLabels: parseLabels(stream.labels), + }; } diff --git a/public/sass/components/_gf-form.scss b/public/sass/components/_gf-form.scss index 0de386f3f68..6d83fc6cf0b 100644 --- a/public/sass/components/_gf-form.scss +++ b/public/sass/components/_gf-form.scss @@ -116,6 +116,11 @@ $input-border: 1px solid $input-border-color; color: $critical; } + &--small { + padding: ($input-padding-y / 2) ($input-padding-x / 2); + font-size: $font-size-xs; + } + &:disabled { color: $text-color-weak; } diff --git a/public/sass/components/_switch.scss b/public/sass/components/_switch.scss index 6eb01ecc32d..c368d8ead67 100644 --- a/public/sass/components/_switch.scss +++ b/public/sass/components/_switch.scss @@ -41,7 +41,6 @@ bottom: 0; right: 0; color: #fff; - font-size: $font-size-sm; text-align: center; font-size: 150%; display: flex; @@ -91,6 +90,20 @@ transform: rotateY(0); } + &--small { + max-width: 2rem; + min-width: 1.5rem; + + input + label { + height: 25px; + } + + input + label::before, + input + label::after { + font-size: $font-size-sm; + } + } + &--table-cell { margin-bottom: 0; margin-right: 0; diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index b70b058879c..70b1901fb50 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -214,7 +214,42 @@ display: grid; grid-column-gap: 1rem; grid-row-gap: 0.1rem; - grid-template-columns: 4px minmax(100px, max-content) 1fr; + grid-template-columns: 4px minmax(100px, max-content) minmax(100px, 25%) 1fr; + font-family: $font-family-monospace; + font-size: 12px; + } + + .logs-controls { + display: flex; + + > * { + margin-right: 1em; + } + } + + .logs-options, + .logs-graph { + margin-bottom: $panel-margin; + } + + .logs-meta { + flex: 1; + color: $text-color-weak; + padding: 2px 0; + } + + .logs-meta-item { + display: inline-block; + margin-right: 1em; + } + + .logs-meta-item__label { + margin-right: 0.5em; + font-size: 0.9em; + font-weight: 500; + } + + .logs-meta-item__value { font-family: $font-family-monospace; } From c1ca1ed35e1441e989787816dd7898cedff60a9d Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Mon, 5 Nov 2018 23:36:58 +0100 Subject: [PATCH 2/7] Time selection via graph --- public/app/features/explore/Explore.tsx | 9 ++++++- public/app/features/explore/Graph.tsx | 28 ++++++++++++++++++---- public/app/features/explore/Logs.tsx | 2 ++ public/app/features/explore/TimePicker.tsx | 28 +++++++++++++++++----- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index 6ec6c79ac5f..78764ab9876 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -898,6 +898,7 @@ export class Explore extends React.PureComponent { height={graphHeight} loading={graphLoading} id={`explore-graph-${position}`} + onChangeTime={this.onChangeTime} range={graphRange} split={split} /> @@ -908,7 +909,13 @@ export class Explore extends React.PureComponent {
) : null} {supportsLogs && showingLogs ? ( - + ) : null} )} diff --git a/public/app/features/explore/Graph.tsx b/public/app/features/explore/Graph.tsx index 6c22ca67509..2c1f08b871d 100644 --- a/public/app/features/explore/Graph.tsx +++ b/public/app/features/explore/Graph.tsx @@ -5,6 +5,7 @@ import { withSize } from 'react-sizeme'; import 'vendor/flot/jquery.flot'; import 'vendor/flot/jquery.flot.time'; +import 'vendor/flot/jquery.flot.selection'; import { RawTimeRange } from 'app/types/series'; import * as dateMath from 'app/core/utils/datemath'; @@ -62,10 +63,10 @@ const FLOT_OPTIONS = { margin: { left: 0, right: 0 }, labelMarginX: 0, }, - // selection: { - // mode: 'x', - // color: '#666', - // }, + selection: { + mode: 'x', + color: '#666', + }, // crosshair: { // mode: 'x', // }, @@ -80,6 +81,7 @@ interface GraphProps { split?: boolean; size?: { width: number; height: number }; userOptions?: any; + onChangeTime?: (range: RawTimeRange) => void; } interface GraphState { @@ -87,6 +89,8 @@ interface GraphState { } export class Graph extends PureComponent { + $el: any; + state = { showAllTimeSeries: false, }; @@ -99,6 +103,8 @@ export class Graph extends PureComponent { componentDidMount() { this.draw(); + this.$el = $(`#${this.props.id}`); + this.$el.bind('plotselected', this.onPlotSelected); } componentDidUpdate(prevProps: GraphProps) { @@ -113,6 +119,20 @@ export class Graph extends PureComponent { } } + componentWillUnmount() { + this.$el.unbind('plotselected', this.onPlotSelected); + } + + onPlotSelected = (event, ranges) => { + if (this.props.onChangeTime) { + const range = { + from: moment(ranges.xaxis.from), + to: moment(ranges.xaxis.to), + }; + this.props.onChangeTime(range); + } + }; + onShowAllTimeSeries = () => { this.setState( { diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 5630d1de8f6..ccfa96bed0b 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -25,6 +25,7 @@ interface LogsProps { loading: boolean; position: string; range?: RawTimeRange; + onChangeTime?: (range: RawTimeRange) => void; } interface LogsState { @@ -88,6 +89,7 @@ export default class Logs extends PureComponent { height="100px" range={range} id={`explore-logs-graph-${position}`} + onChangeTime={this.props.onChangeTime} userOptions={graphOptions} />
diff --git a/public/app/features/explore/TimePicker.tsx b/public/app/features/explore/TimePicker.tsx index 8955fb4aa9b..ed2fd924c78 100644 --- a/public/app/features/explore/TimePicker.tsx +++ b/public/app/features/explore/TimePicker.tsx @@ -16,6 +16,9 @@ export const DEFAULT_RANGE = { * @param value Epoch or relative time */ export function parseTime(value: string, isUtc = false): string { + if (moment.isMoment(value)) { + return value; + } if (value.indexOf('now') !== -1) { return value; } @@ -39,7 +42,8 @@ interface TimePickerState { isOpen: boolean; isUtc: boolean; rangeString: string; - refreshInterval: string; + refreshInterval?: string; + initialRange: RawTimeRange; // Input-controlled text, keep these in a shape that is human-editable fromRaw: string; @@ -49,11 +53,24 @@ interface TimePickerState { export default class TimePicker extends PureComponent { dropdownEl: any; - constructor(props) { - super(props); + state = { + isOpen: false, + isUtc: false, + rangeString: '', + initialRange: DEFAULT_RANGE, + fromRaw: '', + toRaw: '', + refreshInterval: '', + }; + + static getDerivedStateFromProps(props, state) { + if (state.range && state.range === props.range) { + return null; + } const from = props.range ? props.range.from : DEFAULT_RANGE.from; const to = props.range ? props.range.to : DEFAULT_RANGE.to; + const initialRange = props.range || DEFAULT_RANGE; // Ensure internal format const fromRaw = parseTime(from, props.isUtc); @@ -63,13 +80,12 @@ export default class TimePicker extends PureComponent Date: Tue, 6 Nov 2018 11:07:12 +0100 Subject: [PATCH 3/7] Adaptive bar widths for log graph --- public/app/features/explore/Explore.tsx | 3 +- public/app/features/explore/Graph.tsx | 1 + public/app/features/explore/Logs.tsx | 3 ++ public/app/features/explore/TimePicker.tsx | 33 ++++++++++--------- .../plugins/datasource/logging/datasource.ts | 2 +- .../datasource/logging/result_transformer.ts | 4 +-- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index 78764ab9876..896a946a85b 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -475,7 +475,7 @@ export class Explore extends React.PureComponent { from: parseDate(range.from, false), to: parseDate(range.to, true), }; - const { interval } = kbn.calculateInterval(absoluteRange, resolution, datasource.interval); + const { interval, intervalMs } = kbn.calculateInterval(absoluteRange, resolution, datasource.interval); const targets = [ { ...targetOptions, @@ -490,6 +490,7 @@ export class Explore extends React.PureComponent { return { interval, + intervalMs, targets, range: queryRange, }; diff --git a/public/app/features/explore/Graph.tsx b/public/app/features/explore/Graph.tsx index 2c1f08b871d..9e4fea0d3de 100644 --- a/public/app/features/explore/Graph.tsx +++ b/public/app/features/explore/Graph.tsx @@ -6,6 +6,7 @@ import { withSize } from 'react-sizeme'; import 'vendor/flot/jquery.flot'; import 'vendor/flot/jquery.flot.time'; import 'vendor/flot/jquery.flot.selection'; +import 'vendor/flot/jquery.flot.stack'; import { RawTimeRange } from 'app/types/series'; import * as dateMath from 'app/core/utils/datemath'; diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index ccfa96bed0b..edde5acba92 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -12,7 +12,10 @@ const graphOptions = { series: { bars: { show: true, + lineWidth: 5, + // barWidth: 10, }, + // stack: true, }, yaxis: { tickDecimals: 0, diff --git a/public/app/features/explore/TimePicker.tsx b/public/app/features/explore/TimePicker.tsx index ed2fd924c78..a3578263cea 100644 --- a/public/app/features/explore/TimePicker.tsx +++ b/public/app/features/explore/TimePicker.tsx @@ -43,7 +43,7 @@ interface TimePickerState { isUtc: boolean; rangeString: string; refreshInterval?: string; - initialRange: RawTimeRange; + initialRange?: RawTimeRange; // Input-controlled text, keep these in a shape that is human-editable fromRaw: string; @@ -53,24 +53,27 @@ interface TimePickerState { export default class TimePicker extends PureComponent { dropdownEl: any; - state = { - isOpen: false, - isUtc: false, - rangeString: '', - initialRange: DEFAULT_RANGE, - fromRaw: '', - toRaw: '', - refreshInterval: '', - }; + constructor(props) { + super(props); + + this.state = { + isOpen: props.isOpen, + isUtc: props.isUtc, + rangeString: '', + fromRaw: '', + toRaw: '', + initialRange: DEFAULT_RANGE, + refreshInterval: '', + }; + } static getDerivedStateFromProps(props, state) { - if (state.range && state.range === props.range) { - return null; + if (state.initialRange && state.initialRange === props.range) { + return state; } const from = props.range ? props.range.from : DEFAULT_RANGE.from; const to = props.range ? props.range.to : DEFAULT_RANGE.to; - const initialRange = props.range || DEFAULT_RANGE; // Ensure internal format const fromRaw = parseTime(from, props.isUtc); @@ -81,10 +84,10 @@ export default class TimePicker extends PureComponent processStream(stream, DEFAULT_LIMIT)); + const processedStreams = allStreams.map(stream => processStream(stream, DEFAULT_LIMIT, options.intervalMs)); return { data: processedStreams }; }); } diff --git a/public/app/plugins/datasource/logging/result_transformer.ts b/public/app/plugins/datasource/logging/result_transformer.ts index 8aa7ebc12e0..e1e622aeb59 100644 --- a/public/app/plugins/datasource/logging/result_transformer.ts +++ b/public/app/plugins/datasource/logging/result_transformer.ts @@ -143,7 +143,7 @@ export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel { return { meta, series, rows: sortedEntries }; } -export function processStream(stream: LogsStream, limit?: number): LogsStream { +export function processStream(stream: LogsStream, limit?: number, intervalMs?: number): LogsStream { const sortedEntries: any[] = _.chain(stream.entries) .map(entry => processEntry(entry, stream)) .sortBy('timestamp') @@ -155,7 +155,7 @@ export function processStream(stream: LogsStream, limit?: number): LogsStream { let previousTime; const datapoints = sortedEntries.reduce((acc, entry, index) => { // Bucket to nearest minute - const time = Math.round(entry.timeJs / 1000 / 60) * 1000 * 60; + const time = Math.round(entry.timeJs / intervalMs / 10) * intervalMs * 10; // Entry for time if (time === previousTime) { acc[acc.length - 1][0]++; From a5ed86edba52db8500234e9f8de2a62669473ebf Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 6 Nov 2018 12:00:05 +0100 Subject: [PATCH 4/7] Graph log entries by log level --- public/app/core/logs_model.ts | 15 ++++- public/app/core/utils/colors.ts | 20 +++--- .../datasource/logging/result_transformer.ts | 66 +++++++++++-------- 3 files changed, 62 insertions(+), 39 deletions(-) diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index ca7899db7d8..42c2e1ae453 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -1,5 +1,6 @@ import _ from 'lodash'; import { TimeSeries } from 'app/core/core'; +import colors from 'app/core/utils/colors'; export enum LogLevel { crit = 'crit', @@ -9,8 +10,20 @@ export enum LogLevel { info = 'info', debug = 'debug', trace = 'trace', + none = 'none', } +export const LogLevelColor = { + [LogLevel.crit]: colors[7], + [LogLevel.warn]: colors[1], + [LogLevel.err]: colors[4], + [LogLevel.error]: colors[4], + [LogLevel.info]: colors[0], + [LogLevel.debug]: colors[3], + [LogLevel.trace]: colors[3], + [LogLevel.none]: '#eee', +}; + export interface LogSearchMatch { start: number; length: number; @@ -44,7 +57,7 @@ export interface LogsStream { labels: string; entries: LogsStreamEntry[]; parsedLabels: { [key: string]: string }; - graphSeries: TimeSeries; + intervalMs?: number; } export interface LogsStreamEntry { diff --git a/public/app/core/utils/colors.ts b/public/app/core/utils/colors.ts index e8a7366beb5..16214679996 100644 --- a/public/app/core/utils/colors.ts +++ b/public/app/core/utils/colors.ts @@ -10,16 +10,16 @@ export const NO_DATA_COLOR = 'rgba(150, 150, 150, 1)'; export const REGION_FILL_ALPHA = 0.09; const colors = [ - '#7EB26D', - '#EAB839', - '#6ED0E0', - '#EF843C', - '#E24D42', - '#1F78C1', - '#BA43A9', - '#705DA0', - '#508642', - '#CCA300', + '#7EB26D', // 0: pale green + '#EAB839', // 1: mustard + '#6ED0E0', // 2: light blue + '#EF843C', // 3: orange + '#E24D42', // 4: red + '#1F78C1', // 5: ocean + '#BA43A9', // 6: purple + '#705DA0', // 7: violet + '#508642', // 8: dark green + '#CCA300', // 9: dark sand '#447EBC', '#C15C17', '#890F02', diff --git a/public/app/plugins/datasource/logging/result_transformer.ts b/public/app/plugins/datasource/logging/result_transformer.ts index e1e622aeb59..61c0feb493c 100644 --- a/public/app/plugins/datasource/logging/result_transformer.ts +++ b/public/app/plugins/datasource/logging/result_transformer.ts @@ -1,13 +1,12 @@ import _ from 'lodash'; import moment from 'moment'; -import { LogLevel, LogsMetaItem, LogsModel, LogRow, LogsStream } from 'app/core/logs_model'; +import { LogLevel, LogLevelColor, LogsMetaItem, LogsModel, LogRow, LogsStream } from 'app/core/logs_model'; import { TimeSeries } from 'app/core/core'; -import colors from 'app/core/utils/colors'; export function getLogLevel(line: string): LogLevel { if (!line) { - return undefined; + return LogLevel.none; } let level: LogLevel; Object.keys(LogLevel).forEach(key => { @@ -18,6 +17,9 @@ export function getLogLevel(line: string): LogLevel { } } }); + if (!level) { + level = LogLevel.none; + } return level; } @@ -107,8 +109,13 @@ export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel { }, ]; + let intervalMs; + // Flatten entries of streams - const combinedEntries = streams.reduce((acc, stream) => { + const combinedEntries: LogRow[] = streams.reduce((acc, stream) => { + // Set interval for graphs + intervalMs = stream.intervalMs; + // Overwrite labels to be only the non-common ones const labels = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels)); return [ @@ -120,15 +127,34 @@ export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel { ]; }, []); - const commonLabelsAlias = - streams.length === 1 ? formatLabels(commonLabels) : `Stream with common labels ${formatLabels(commonLabels)}`; - const series = streams.map((stream, index) => { - const colorIndex = index % colors.length; - stream.graphSeries.setColor(colors[colorIndex]); - stream.graphSeries.alias = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels), commonLabelsAlias); - return stream.graphSeries; + // Graph time series by log level + const seriesByLevel = {}; + combinedEntries.forEach(entry => { + if (!seriesByLevel[entry.logLevel]) { + seriesByLevel[entry.logLevel] = { lastTs: null, datapoints: [], alias: entry.logLevel }; + } + const levelSeries = seriesByLevel[entry.logLevel]; + + // Bucket to nearest minute + const time = Math.round(entry.timeJs / intervalMs / 10) * intervalMs * 10; + // Entry for time + if (time === levelSeries.lastTs) { + levelSeries.datapoints[levelSeries.datapoints.length - 1][0]++; + } else { + levelSeries.datapoints.push([1, time]); + levelSeries.lastTs = time; + } }); + const series = Object.keys(seriesByLevel).reduce((acc, level, index) => { + if (seriesByLevel[level]) { + const gs = new TimeSeries(seriesByLevel[level]); + gs.setColor(LogLevelColor[level]); + acc.push(gs); + } + return acc; + }, []); + const sortedEntries = _.chain(combinedEntries) .sortBy('timestamp') .reverse() @@ -151,25 +177,9 @@ export function processStream(stream: LogsStream, limit?: number, intervalMs?: n .slice(0, limit || stream.entries.length) .value(); - // Build graph data - let previousTime; - const datapoints = sortedEntries.reduce((acc, entry, index) => { - // Bucket to nearest minute - const time = Math.round(entry.timeJs / intervalMs / 10) * intervalMs * 10; - // Entry for time - if (time === previousTime) { - acc[acc.length - 1][0]++; - } else { - acc.push([1, time]); - previousTime = time; - } - return acc; - }, []); - const graphSeries = new TimeSeries({ datapoints, alias: stream.labels }); - return { ...stream, - graphSeries, + intervalMs, entries: sortedEntries, parsedLabels: parseLabels(stream.labels), }; From f5575459ebe76c56ba4ea6a5bdf88f7930775a6f Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 6 Nov 2018 12:05:10 +0100 Subject: [PATCH 5/7] unify log level colors between rows and graph --- public/sass/pages/_explore.scss | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 70b1901fb50..0ca6d17fcf0 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -270,18 +270,26 @@ opacity: 0.8; } - .logs-row-level-crit, + .logs-row-level-crit { + background-color: #705da0; + } + .logs-row-level-error, .logs-row-level-err { - background-color: $red; + background-color: #e24d42; } .logs-row-level-warn { - background-color: $orange; + background-color: #eab839; } .logs-row-level-info { - background-color: $green; + background-color: #7eb26d; + } + + .logs-row-level-trace, + .logs-row-level-debug { + background-color: #1f78c1; } } } From dfc0c5052d79e06f03270897d825447bc5993945 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 6 Nov 2018 15:40:29 +0100 Subject: [PATCH 6/7] Fix loglevel tests for Explore loggging --- .../app/plugins/datasource/logging/result_transformer.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/logging/result_transformer.test.ts b/public/app/plugins/datasource/logging/result_transformer.test.ts index 28debe41585..996edc56261 100644 --- a/public/app/plugins/datasource/logging/result_transformer.test.ts +++ b/public/app/plugins/datasource/logging/result_transformer.test.ts @@ -4,11 +4,11 @@ import { findCommonLabels, findUncommonLabels, formatLabels, getLogLevel, parseL describe('getLoglevel()', () => { it('returns no log level on empty line', () => { - expect(getLogLevel('')).toBe(undefined); + expect(getLogLevel('')).toBe(LogLevel.none); }); it('returns no log level on when level is part of a word', () => { - expect(getLogLevel('this is a warning')).toBe(undefined); + expect(getLogLevel('this is a warning')).toBe(LogLevel.none); }); it('returns log level on line contains a log level', () => { From 6b8d7c89ac1fa867e394638bc542bfc4e84acadb Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Thu, 8 Nov 2018 14:24:54 +0100 Subject: [PATCH 7/7] Refactored log stream merging, added types, tests, comments --- public/app/core/logs_model.ts | 50 ++++- public/app/features/explore/Explore.tsx | 22 ++- .../plugins/datasource/logging/datasource.ts | 23 +-- .../logging/result_transformer.test.ts | 91 ++++++++- .../datasource/logging/result_transformer.ts | 179 +++++++++--------- 5 files changed, 241 insertions(+), 124 deletions(-) diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 42c2e1ae453..ab0a3f26a88 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -31,15 +31,16 @@ export interface LogSearchMatch { } export interface LogRow { - key: string; entry: string; + key: string; // timestamp + labels labels: string; logLevel: LogLevel; - timestamp: string; - timeFromNow: string; - timeJs: number; - timeLocal: string; searchWords?: string[]; + timestamp: string; // ISO with nanosec precision + timeFromNow: string; + timeEpochMs: number; + timeLocal: string; + uniqueLabels?: string; } export interface LogsMetaItem { @@ -56,11 +57,46 @@ export interface LogsModel { export interface LogsStream { labels: string; entries: LogsStreamEntry[]; - parsedLabels: { [key: string]: string }; - intervalMs?: number; + search?: string; + parsedLabels?: LogsStreamLabels; + uniqueLabels?: string; } export interface LogsStreamEntry { line: string; timestamp: string; } + +export interface LogsStreamLabels { + [key: string]: string; +} + +export function makeSeriesForLogs(rows: LogRow[], intervalMs: number): TimeSeries[] { + // Graph time series by log level + const seriesByLevel = {}; + rows.forEach(row => { + if (!seriesByLevel[row.logLevel]) { + seriesByLevel[row.logLevel] = { lastTs: null, datapoints: [], alias: row.logLevel }; + } + const levelSeries = seriesByLevel[row.logLevel]; + + // Bucket to nearest minute + const time = Math.round(row.timeEpochMs / intervalMs / 10) * intervalMs * 10; + // Entry for time + if (time === levelSeries.lastTs) { + levelSeries.datapoints[levelSeries.datapoints.length - 1][0]++; + } else { + levelSeries.datapoints.push([1, time]); + levelSeries.lastTs = time; + } + }); + + return Object.keys(seriesByLevel).reduce((acc, level) => { + if (seriesByLevel[level]) { + const gs = new TimeSeries(seriesByLevel[level]); + gs.setColor(LogLevelColor[level]); + acc.push(gs); + } + return acc; + }, []); +} diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index 896a946a85b..753f158fd9f 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -28,6 +28,17 @@ import { DataSource } from 'app/types/datasources'; const MAX_HISTORY_ITEMS = 100; +function getIntervals(range: RawTimeRange, datasource, resolution: number): { interval: string; intervalMs: number } { + if (!datasource || !resolution) { + return { interval: '1s', intervalMs: 1000 }; + } + const absoluteRange: RawTimeRange = { + from: parseDate(range.from, false), + to: parseDate(range.to, true), + }; + return kbn.calculateInterval(absoluteRange, resolution, datasource.interval); +} + function makeTimeSeriesList(dataList, options) { return dataList.map((seriesData, index) => { const datapoints = seriesData.datapoints || []; @@ -470,12 +481,7 @@ export class Explore extends React.PureComponent { targetOptions: { format: string; hinting?: boolean; instant?: boolean } ) { const { datasource, range } = this.state; - const resolution = this.el.offsetWidth; - const absoluteRange: RawTimeRange = { - from: parseDate(range.from, false), - to: parseDate(range.to, true), - }; - const { interval, intervalMs } = kbn.calculateInterval(absoluteRange, resolution, datasource.interval); + const { interval, intervalMs } = getIntervals(range, datasource, this.el.offsetWidth); const targets = [ { ...targetOptions, @@ -759,6 +765,7 @@ export class Explore extends React.PureComponent { const tableButtonActive = showingBoth || showingTable ? 'active' : ''; const exploreClass = split ? 'explore explore-split' : 'explore'; const selectedDatasource = datasource ? exploreDatasources.find(d => d.label === datasource.name) : undefined; + const graphRangeIntervals = getIntervals(graphRange, datasource, this.el ? this.el.offsetWidth : 0); const graphLoading = queryTransactions.some(qt => qt.resultType === 'Graph' && !qt.done); const tableLoading = queryTransactions.some(qt => qt.resultType === 'Table' && !qt.done); const logsLoading = queryTransactions.some(qt => qt.resultType === 'Logs' && !qt.done); @@ -775,7 +782,8 @@ export class Explore extends React.PureComponent { ? datasource.mergeStreams( _.flatten( queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result) - ) + ), + graphRangeIntervals.intervalMs ) : undefined; const loading = queryTransactions.some(qt => !qt.done); diff --git a/public/app/plugins/datasource/logging/datasource.ts b/public/app/plugins/datasource/logging/datasource.ts index 20bacacbd3a..fcf3028c025 100644 --- a/public/app/plugins/datasource/logging/datasource.ts +++ b/public/app/plugins/datasource/logging/datasource.ts @@ -3,10 +3,10 @@ import _ from 'lodash'; import * as dateMath from 'app/core/utils/datemath'; import LanguageProvider from './language_provider'; -import { mergeStreams, processStream } from './result_transformer'; -import { LogsStream } from 'app/core/logs_model'; +import { mergeStreamsToLogs } from './result_transformer'; +import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model'; -const DEFAULT_LIMIT = 1000; +export const DEFAULT_LIMIT = 1000; const DEFAULT_QUERY_PARAMS = { direction: 'BACKWARD', @@ -68,8 +68,10 @@ export default class LoggingDatasource { return this.backendSrv.datasourceRequest(req); } - mergeStreams(streams: LogsStream[]) { - return mergeStreams(streams, DEFAULT_LIMIT); + mergeStreams(streams: LogsStream[], intervalMs: number): LogsModel { + const logs = mergeStreamsToLogs(streams); + logs.series = makeSeriesForLogs(logs.rows, intervalMs); + return logs; } prepareQueryTarget(target, options) { @@ -84,7 +86,7 @@ export default class LoggingDatasource { }; } - query(options) { + query(options): Promise<{ data: LogsStream[] }> { const queryTargets = options.targets .filter(target => target.expr) .map(target => this.prepareQueryTarget(target, options)); @@ -96,17 +98,16 @@ export default class LoggingDatasource { return Promise.all(queries).then((results: any[]) => { // Flatten streams from multiple queries - const allStreams = results.reduce((acc, response, i) => { - const streams = response.data.streams || []; + const allStreams: LogsStream[] = results.reduce((acc, response, i) => { + const streams: LogsStream[] = response.data.streams || []; // Inject search for match highlighting - const search = queryTargets[i].regexp; + const search: string = queryTargets[i].regexp; streams.forEach(s => { s.search = search; }); return [...acc, ...streams]; }, []); - const processedStreams = allStreams.map(stream => processStream(stream, DEFAULT_LIMIT, options.intervalMs)); - return { data: processedStreams }; + return { data: allStreams }; }); } diff --git a/public/app/plugins/datasource/logging/result_transformer.test.ts b/public/app/plugins/datasource/logging/result_transformer.test.ts index 996edc56261..6f88c301fcd 100644 --- a/public/app/plugins/datasource/logging/result_transformer.test.ts +++ b/public/app/plugins/datasource/logging/result_transformer.test.ts @@ -1,6 +1,13 @@ -import { LogLevel } from 'app/core/logs_model'; +import { LogLevel, LogsStream } from 'app/core/logs_model'; -import { findCommonLabels, findUncommonLabels, formatLabels, getLogLevel, parseLabels } from './result_transformer'; +import { + findCommonLabels, + findUniqueLabels, + formatLabels, + getLogLevel, + mergeStreamsToLogs, + parseLabels, +} from './result_transformer'; describe('getLoglevel()', () => { it('returns no log level on empty line', () => { @@ -61,16 +68,88 @@ describe('findCommonLabels()', () => { }); }); -describe('findUncommonLabels()', () => { +describe('findUniqueLabels()', () => { it('returns no uncommon labels on empty sets', () => { - expect(findUncommonLabels({}, {})).toEqual({}); + expect(findUniqueLabels({}, {})).toEqual({}); }); it('returns all labels given no common labels', () => { - expect(findUncommonLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' }); + expect(findUniqueLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' }); }); it('returns all labels except the common labels', () => { - expect(findUncommonLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' }); + expect(findUniqueLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' }); + }); +}); + +describe('mergeStreamsToLogs()', () => { + it('returns empty logs given no streams', () => { + expect(mergeStreamsToLogs([]).rows).toEqual([]); + }); + + it('returns processed logs from single stream', () => { + const stream1: LogsStream = { + labels: '{foo="bar"}', + entries: [ + { + line: 'WARN boooo', + timestamp: '1970-01-01T00:00:00Z', + }, + ], + }; + expect(mergeStreamsToLogs([stream1]).rows).toMatchObject([ + { + entry: 'WARN boooo', + labels: '{foo="bar"}', + key: 'EK1970-01-01T00:00:00Z{foo="bar"}', + logLevel: 'warn', + uniqueLabels: '', + }, + ]); + }); + + it('returns merged logs from multiple streams sorted by time and with unique labels', () => { + const stream1: LogsStream = { + labels: '{foo="bar", baz="1"}', + entries: [ + { + line: 'WARN boooo', + timestamp: '1970-01-01T00:00:01Z', + }, + ], + }; + const stream2: LogsStream = { + labels: '{foo="bar", baz="2"}', + entries: [ + { + line: 'INFO 1', + timestamp: '1970-01-01T00:00:00Z', + }, + { + line: 'INFO 2', + timestamp: '1970-01-01T00:00:02Z', + }, + ], + }; + expect(mergeStreamsToLogs([stream1, stream2]).rows).toMatchObject([ + { + entry: 'INFO 2', + labels: '{foo="bar", baz="2"}', + logLevel: 'info', + uniqueLabels: '{baz="2"}', + }, + { + entry: 'WARN boooo', + labels: '{foo="bar", baz="1"}', + logLevel: 'warn', + uniqueLabels: '{baz="1"}', + }, + { + entry: 'INFO 1', + labels: '{foo="bar", baz="2"}', + logLevel: 'info', + uniqueLabels: '{baz="2"}', + }, + ]); }); }); diff --git a/public/app/plugins/datasource/logging/result_transformer.ts b/public/app/plugins/datasource/logging/result_transformer.ts index 61c0feb493c..3d04deb7ed2 100644 --- a/public/app/plugins/datasource/logging/result_transformer.ts +++ b/public/app/plugins/datasource/logging/result_transformer.ts @@ -1,9 +1,23 @@ import _ from 'lodash'; import moment from 'moment'; -import { LogLevel, LogLevelColor, LogsMetaItem, LogsModel, LogRow, LogsStream } from 'app/core/logs_model'; -import { TimeSeries } from 'app/core/core'; +import { + LogLevel, + LogsMetaItem, + LogsModel, + LogRow, + LogsStream, + LogsStreamEntry, + LogsStreamLabels, +} from 'app/core/logs_model'; +import { DEFAULT_LIMIT } from './datasource'; +/** + * Returns the log level of a log line. + * Parse the line for level words. If no level is found, it returns `LogLevel.none`. + * + * Example: `getLogLevel('WARN 1999-12-31 this is great') // LogLevel.warn` + */ export function getLogLevel(line: string): LogLevel { if (!line) { return LogLevel.none; @@ -23,9 +37,18 @@ export function getLogLevel(line: string): LogLevel { return level; } +/** + * Regexp to extract Prometheus-style labels + */ const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g; -export function parseLabels(labels: string): { [key: string]: string } { - const labelsByKey = {}; + +/** + * Returns a map of label keys to value from an input selector string. + * + * Example: `parseLabels('{job="foo", instance="bar"}) // {job: "foo", instance: "bar"}` + */ +export function parseLabels(labels: string): LogsStreamLabels { + const labelsByKey: LogsStreamLabels = {}; labels.replace(labelRegexp, (_, key, operator, value) => { labelsByKey[key] = value; return ''; @@ -33,7 +56,10 @@ export function parseLabels(labels: string): { [key: string]: string } { return labelsByKey; } -export function findCommonLabels(labelsSets: any[]) { +/** + * Returns a map labels that are common to the given label sets. + */ +export function findCommonLabels(labelsSets: LogsStreamLabels[]): LogsStreamLabels { return labelsSets.reduce((acc, labels) => { if (!labels) { throw new Error('Need parsed labels to find common labels.'); @@ -59,15 +85,21 @@ export function findCommonLabels(labelsSets: any[]) { }, undefined); } -export function findUncommonLabels(labels, commonLabels) { - const uncommonLabels = { ...labels }; +/** + * Returns a map of labels that are in `labels`, but not in `commonLabels`. + */ +export function findUniqueLabels(labels: LogsStreamLabels, commonLabels: LogsStreamLabels): LogsStreamLabels { + const uncommonLabels: LogsStreamLabels = { ...labels }; Object.keys(commonLabels).forEach(key => { delete uncommonLabels[key]; }); return uncommonLabels; } -export function formatLabels(labels, defaultValue = '') { +/** + * Serializes the given labels to a string. + */ +export function formatLabels(labels: LogsStreamLabels, defaultValue = ''): string { if (!labels || Object.keys(labels).length === 0) { return defaultValue; } @@ -76,111 +108,72 @@ export function formatLabels(labels, defaultValue = '') { return ['{', cleanSelector, '}'].join(''); } -export function processEntry(entry: { line: string; timestamp: string }, stream): LogRow { +export function processEntry(entry: LogsStreamEntry, labels: string, uniqueLabels: string, search: string): LogRow { const { line, timestamp } = entry; - const { labels } = stream; + // Assumes unique-ness, needs nanosec precision for timestamp const key = `EK${timestamp}${labels}`; const time = moment(timestamp); - const timeJs = time.valueOf(); + const timeEpochMs = time.valueOf(); const timeFromNow = time.fromNow(); const timeLocal = time.format('YYYY-MM-DD HH:mm:ss'); const logLevel = getLogLevel(line); return { key, + labels, logLevel, timeFromNow, - timeJs, + timeEpochMs, timeLocal, + uniqueLabels, entry: line, - labels: formatLabels(labels), - searchWords: [stream.search], + searchWords: search ? [search] : [], timestamp: timestamp, }; } -export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel { - // Find meta data - const commonLabels = findCommonLabels(streams.map(stream => stream.parsedLabels)); - const meta: LogsMetaItem[] = [ - { +export function mergeStreamsToLogs(streams: LogsStream[], limit = DEFAULT_LIMIT): LogsModel { + // Find unique labels for each stream + streams = streams.map(stream => ({ + ...stream, + parsedLabels: parseLabels(stream.labels), + })); + const commonLabels = findCommonLabels(streams.map(model => model.parsedLabels)); + streams = streams.map(stream => ({ + ...stream, + uniqueLabels: formatLabels(findUniqueLabels(stream.parsedLabels, commonLabels)), + })); + + // Merge stream entries into single list of log rows + const sortedRows: LogRow[] = _.chain(streams) + .reduce( + (acc: LogRow[], stream: LogsStream) => [ + ...acc, + ...stream.entries.map(entry => processEntry(entry, stream.labels, stream.uniqueLabels, stream.search)), + ], + [] + ) + .sortBy('timestamp') + .reverse() + .value(); + + // Meta data to display in status + const meta: LogsMetaItem[] = []; + if (_.size(commonLabels) > 0) { + meta.push({ label: 'Common labels', value: formatLabels(commonLabels), - }, - ]; - - let intervalMs; - - // Flatten entries of streams - const combinedEntries: LogRow[] = streams.reduce((acc, stream) => { - // Set interval for graphs - intervalMs = stream.intervalMs; - - // Overwrite labels to be only the non-common ones - const labels = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels)); - return [ - ...acc, - ...stream.entries.map(entry => ({ - ...entry, - labels, - })), - ]; - }, []); - - // Graph time series by log level - const seriesByLevel = {}; - combinedEntries.forEach(entry => { - if (!seriesByLevel[entry.logLevel]) { - seriesByLevel[entry.logLevel] = { lastTs: null, datapoints: [], alias: entry.logLevel }; - } - const levelSeries = seriesByLevel[entry.logLevel]; - - // Bucket to nearest minute - const time = Math.round(entry.timeJs / intervalMs / 10) * intervalMs * 10; - // Entry for time - if (time === levelSeries.lastTs) { - levelSeries.datapoints[levelSeries.datapoints.length - 1][0]++; - } else { - levelSeries.datapoints.push([1, time]); - levelSeries.lastTs = time; - } - }); - - const series = Object.keys(seriesByLevel).reduce((acc, level, index) => { - if (seriesByLevel[level]) { - const gs = new TimeSeries(seriesByLevel[level]); - gs.setColor(LogLevelColor[level]); - acc.push(gs); - } - return acc; - }, []); - - const sortedEntries = _.chain(combinedEntries) - .sortBy('timestamp') - .reverse() - .slice(0, limit || combinedEntries.length) - .value(); - - meta.push({ - label: 'Limit', - value: `${limit} (${sortedEntries.length} returned)`, - }); - - return { meta, series, rows: sortedEntries }; -} - -export function processStream(stream: LogsStream, limit?: number, intervalMs?: number): LogsStream { - const sortedEntries: any[] = _.chain(stream.entries) - .map(entry => processEntry(entry, stream)) - .sortBy('timestamp') - .reverse() - .slice(0, limit || stream.entries.length) - .value(); + }); + } + if (limit) { + meta.push({ + label: 'Limit', + value: `${limit} (${sortedRows.length} returned)`, + }); + } return { - ...stream, - intervalMs, - entries: sortedEntries, - parsedLabels: parseLabels(stream.labels), + meta, + rows: sortedRows, }; }