From 583334df051ca736140663256c6bed2303c34f84 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 2 Nov 2018 08:25:36 +0100 Subject: [PATCH] Explore: Logging graph overview and view options - Logging gets a graph for log distribution (currently per stream, but I think I'll change that to per log-level) - added grid columns for timestamp and unique labels - show common labels of streams - View options to show/hide time columns, label columns - created `--small` modifier for Switch CSS classes - merging of streams is now a datasource responsibility --- public/app/core/components/Switch/Switch.tsx | 11 +- public/app/core/logs_model.ts | 34 +++-- public/app/features/explore/Explore.tsx | 16 ++- public/app/features/explore/Graph.tsx | 5 +- public/app/features/explore/Logs.tsx | 109 ++++++++++++++- .../plugins/datasource/logging/datasource.ts | 13 +- .../logging/result_transformer.test.ts | 56 +++++++- .../datasource/logging/result_transformer.ts | 132 +++++++++++++++++- public/sass/components/_gf-form.scss | 5 + public/sass/components/_switch.scss | 15 +- public/sass/pages/_explore.scss | 37 ++++- 11 files changed, 397 insertions(+), 36 deletions(-) diff --git a/public/app/core/components/Switch/Switch.tsx b/public/app/core/components/Switch/Switch.tsx index ba09267ebd2..a4bb73a291b 100644 --- a/public/app/core/components/Switch/Switch.tsx +++ b/public/app/core/components/Switch/Switch.tsx @@ -5,6 +5,7 @@ export interface Props { label: string; checked: boolean; labelClass?: string; + small?: boolean; switchClass?: string; onChange: (event) => any; } @@ -24,10 +25,14 @@ export class Switch extends PureComponent { }; render() { - const { labelClass, switchClass, label, checked } = this.props; + const { labelClass = '', switchClass = '', label, checked, small } = this.props; const labelId = `check-${this.state.id}`; - const labelClassName = `gf-form-label ${labelClass} pointer`; - const switchClassName = `gf-form-switch ${switchClass}`; + let labelClassName = `gf-form-label ${labelClass} pointer`; + let switchClassName = `gf-form-switch ${switchClass}`; + if (small) { + labelClassName += ' gf-form-label--small'; + switchClassName += ' gf-form-switch--small'; + } return (
diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index e6f317dbeb7..ca7899db7d8 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { TimeSeries } from 'app/core/core'; export enum LogLevel { crit = 'crit', @@ -19,25 +20,34 @@ export interface LogSearchMatch { export interface LogRow { key: string; entry: string; + labels: string; logLevel: LogLevel; timestamp: string; timeFromNow: string; + timeJs: number; timeLocal: string; searchWords?: string[]; } -export interface LogsModel { - rows: LogRow[]; +export interface LogsMetaItem { + label: string; + value: string; } -export function mergeStreams(streams: LogsModel[], limit?: number): LogsModel { - const combinedEntries = streams.reduce((acc, stream) => { - return [...acc, ...stream.rows]; - }, []); - const sortedEntries = _.chain(combinedEntries) - .sortBy('timestamp') - .reverse() - .slice(0, limit || combinedEntries.length) - .value(); - return { rows: sortedEntries }; +export interface LogsModel { + meta?: LogsMetaItem[]; + rows: LogRow[]; + series?: TimeSeries[]; +} + +export interface LogsStream { + labels: string; + entries: LogsStreamEntry[]; + parsedLabels: { [key: string]: string }; + graphSeries: TimeSeries; +} + +export interface LogsStreamEntry { + line: string; + timestamp: string; } diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index af771bad5dd..6ec6c79ac5f 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -25,7 +25,6 @@ import ErrorBoundary from './ErrorBoundary'; import TimePicker from './TimePicker'; import { ensureQueries, generateQueryKey, hasQuery } from './utils/query'; import { DataSource } from 'app/types/datasources'; -import { mergeStreams } from 'app/core/logs_model'; const MAX_HISTORY_ITEMS = 100; @@ -770,9 +769,14 @@ export class Explore extends React.PureComponent { new TableModel(), ...queryTransactions.filter(qt => qt.resultType === 'Table' && qt.done && qt.result).map(qt => qt.result) ); - const logsResult = mergeStreams( - queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result) - ); + const logsResult = + datasource && datasource.mergeStreams + ? datasource.mergeStreams( + _.flatten( + queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result) + ) + ) + : undefined; const loading = queryTransactions.some(qt => !qt.done); const showStartPages = StartPage && queryTransactions.length === 0; const viewModeCount = [supportsGraph, supportsLogs, supportsTable].filter(m => m).length; @@ -903,7 +907,9 @@ export class Explore extends React.PureComponent { ) : null} - {supportsLogs && showingLogs ? : null} + {supportsLogs && showingLogs ? ( + + ) : null} )} diff --git a/public/app/features/explore/Graph.tsx b/public/app/features/explore/Graph.tsx index bbb055067a2..6c22ca67509 100644 --- a/public/app/features/explore/Graph.tsx +++ b/public/app/features/explore/Graph.tsx @@ -79,6 +79,7 @@ interface GraphProps { range: RawTimeRange; split?: boolean; size?: { width: number; height: number }; + userOptions?: any; } interface GraphState { @@ -122,7 +123,7 @@ export class Graph extends PureComponent { }; draw() { - const { range, size } = this.props; + const { range, size, userOptions = {} } = this.props; const data = this.getGraphData(); const $el = $(`#${this.props.id}`); @@ -153,12 +154,14 @@ export class Graph extends PureComponent { max: max, label: 'Datetime', ticks: ticks, + timezone: 'browser', timeformat: time_format(ticks, min, max), }, }; const options = { ...FLOT_OPTIONS, ...dynamicOptions, + ...userOptions, }; $.plot($el, series, options); } diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 278c5ee016d..5630d1de8f6 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -1,29 +1,130 @@ import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; +import { RawTimeRange } from 'app/types/series'; import { LogsModel } from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; +import { Switch } from 'app/core/components/Switch/Switch'; + +import Graph from './Graph'; + +const graphOptions = { + series: { + bars: { + show: true, + }, + }, + yaxis: { + tickDecimals: 0, + }, +}; interface LogsProps { className?: string; data: LogsModel; loading: boolean; + position: string; + range?: RawTimeRange; } -export default class Logs extends PureComponent { +interface LogsState { + showLabels: boolean; + showLocalTime: boolean; + showUtc: boolean; +} + +export default class Logs extends PureComponent { + state = { + showLabels: true, + showLocalTime: true, + showUtc: false, + }; + + onChangeLabels = (event: React.SyntheticEvent) => { + const target = event.target as HTMLInputElement; + this.setState({ + showLabels: target.checked, + }); + }; + + onChangeLocalTime = (event: React.SyntheticEvent) => { + const target = event.target as HTMLInputElement; + this.setState({ + showLocalTime: target.checked, + }); + }; + + onChangeUtc = (event: React.SyntheticEvent) => { + const target = event.target as HTMLInputElement; + this.setState({ + showUtc: target.checked, + }); + }; + render() { - const { className = '', data, loading = false } = this.props; + const { className = '', data, loading = false, position, range } = this.props; + const { showLabels, showLocalTime, showUtc } = this.state; const hasData = data && data.rows && data.rows.length > 0; + const cssColumnSizes = ['4px']; + if (showUtc) { + cssColumnSizes.push('minmax(100px, max-content)'); + } + if (showLocalTime) { + cssColumnSizes.push('minmax(100px, max-content)'); + } + if (showLabels) { + cssColumnSizes.push('minmax(100px, 25%)'); + } + cssColumnSizes.push('1fr'); + const logEntriesStyle = { + gridTemplateColumns: cssColumnSizes.join(' '), + }; + return (
+
+ +
+ +
+
+ + + + {hasData && + data.meta && ( +
+ {data.meta.map(item => ( +
+ {item.label}: + {item.value} +
+ ))} +
+ )} +
+
+
{loading &&
} -
+
{hasData && data.rows.map(row => (
-
{row.timeLocal}
+ {showUtc &&
{row.timestamp}
} + {showLocalTime &&
{row.timeLocal}
} + {showLabels && ( +
+ {row.labels} +
+ )}
processStream(stream, DEFAULT_LIMIT)); + return { data: processedStreams }; }); } diff --git a/public/app/plugins/datasource/logging/result_transformer.test.ts b/public/app/plugins/datasource/logging/result_transformer.test.ts index c1e6913a388..28debe41585 100644 --- a/public/app/plugins/datasource/logging/result_transformer.test.ts +++ b/public/app/plugins/datasource/logging/result_transformer.test.ts @@ -1,6 +1,6 @@ import { LogLevel } from 'app/core/logs_model'; -import { getLogLevel } from './result_transformer'; +import { findCommonLabels, findUncommonLabels, formatLabels, getLogLevel, parseLabels } from './result_transformer'; describe('getLoglevel()', () => { it('returns no log level on empty line', () => { @@ -20,3 +20,57 @@ describe('getLoglevel()', () => { expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn); }); }); + +describe('parseLabels()', () => { + it('returns no labels on emtpy labels string', () => { + expect(parseLabels('')).toEqual({}); + expect(parseLabels('{}')).toEqual({}); + }); + + it('returns labels on labels string', () => { + expect(parseLabels('{foo="bar", baz="42"}')).toEqual({ foo: '"bar"', baz: '"42"' }); + }); +}); + +describe('formatLabels()', () => { + it('returns no labels on emtpy label set', () => { + expect(formatLabels({})).toEqual(''); + expect(formatLabels({}, 'foo')).toEqual('foo'); + }); + + it('returns label string on label set', () => { + expect(formatLabels({ foo: '"bar"', baz: '"42"' })).toEqual('{baz="42", foo="bar"}'); + }); +}); + +describe('findCommonLabels()', () => { + it('returns no common labels on empty sets', () => { + expect(findCommonLabels([{}])).toEqual({}); + expect(findCommonLabels([{}, {}])).toEqual({}); + }); + + it('returns no common labels on differing sets', () => { + expect(findCommonLabels([{ foo: '"bar"' }, {}])).toEqual({}); + expect(findCommonLabels([{}, { foo: '"bar"' }])).toEqual({}); + expect(findCommonLabels([{ baz: '42' }, { foo: '"bar"' }])).toEqual({}); + expect(findCommonLabels([{ foo: '42', baz: '"bar"' }, { foo: '"bar"' }])).toEqual({}); + }); + + it('returns the single labels set as common labels', () => { + expect(findCommonLabels([{ foo: '"bar"' }])).toEqual({ foo: '"bar"' }); + }); +}); + +describe('findUncommonLabels()', () => { + it('returns no uncommon labels on empty sets', () => { + expect(findUncommonLabels({}, {})).toEqual({}); + }); + + it('returns all labels given no common labels', () => { + expect(findUncommonLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' }); + }); + + it('returns all labels except the common labels', () => { + expect(findUncommonLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' }); + }); +}); diff --git a/public/app/plugins/datasource/logging/result_transformer.ts b/public/app/plugins/datasource/logging/result_transformer.ts index 526a9c7da2c..8aa7ebc12e0 100644 --- a/public/app/plugins/datasource/logging/result_transformer.ts +++ b/public/app/plugins/datasource/logging/result_transformer.ts @@ -1,7 +1,9 @@ import _ from 'lodash'; import moment from 'moment'; -import { LogLevel, LogsModel, LogRow } from 'app/core/logs_model'; +import { LogLevel, LogsMetaItem, LogsModel, LogRow, LogsStream } from 'app/core/logs_model'; +import { TimeSeries } from 'app/core/core'; +import colors from 'app/core/utils/colors'; export function getLogLevel(line: string): LogLevel { if (!line) { @@ -19,11 +21,65 @@ export function getLogLevel(line: string): LogLevel { return level; } +const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g; +export function parseLabels(labels: string): { [key: string]: string } { + const labelsByKey = {}; + labels.replace(labelRegexp, (_, key, operator, value) => { + labelsByKey[key] = value; + return ''; + }); + return labelsByKey; +} + +export function findCommonLabels(labelsSets: any[]) { + return labelsSets.reduce((acc, labels) => { + if (!labels) { + throw new Error('Need parsed labels to find common labels.'); + } + if (!acc) { + // Initial set + acc = { ...labels }; + } else { + // Remove incoming labels that are missing or not matching in value + Object.keys(labels).forEach(key => { + if (acc[key] === undefined || acc[key] !== labels[key]) { + delete acc[key]; + } + }); + // Remove common labels that are missing from incoming label set + Object.keys(acc).forEach(key => { + if (labels[key] === undefined) { + delete acc[key]; + } + }); + } + return acc; + }, undefined); +} + +export function findUncommonLabels(labels, commonLabels) { + const uncommonLabels = { ...labels }; + Object.keys(commonLabels).forEach(key => { + delete uncommonLabels[key]; + }); + return uncommonLabels; +} + +export function formatLabels(labels, defaultValue = '') { + if (!labels || Object.keys(labels).length === 0) { + return defaultValue; + } + const labelKeys = Object.keys(labels).sort(); + const cleanSelector = labelKeys.map(key => `${key}=${labels[key]}`).join(', '); + return ['{', cleanSelector, '}'].join(''); +} + export function processEntry(entry: { line: string; timestamp: string }, stream): LogRow { const { line, timestamp } = entry; const { labels } = stream; const key = `EK${timestamp}${labels}`; const time = moment(timestamp); + const timeJs = time.valueOf(); const timeFromNow = time.fromNow(); const timeLocal = time.format('YYYY-MM-DD HH:mm:ss'); const logLevel = getLogLevel(line); @@ -32,21 +88,89 @@ export function processEntry(entry: { line: string; timestamp: string }, stream) key, logLevel, timeFromNow, + timeJs, timeLocal, entry: line, + labels: formatLabels(labels), searchWords: [stream.search], timestamp: timestamp, }; } -export function processStreams(streams, limit?: number): LogsModel { +export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel { + // Find meta data + const commonLabels = findCommonLabels(streams.map(stream => stream.parsedLabels)); + const meta: LogsMetaItem[] = [ + { + label: 'Common labels', + value: formatLabels(commonLabels), + }, + ]; + + // Flatten entries of streams const combinedEntries = streams.reduce((acc, stream) => { - return [...acc, ...stream.entries.map(entry => processEntry(entry, stream))]; + // Overwrite labels to be only the non-common ones + const labels = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels)); + return [ + ...acc, + ...stream.entries.map(entry => ({ + ...entry, + labels, + })), + ]; }, []); + + const commonLabelsAlias = + streams.length === 1 ? formatLabels(commonLabels) : `Stream with common labels ${formatLabels(commonLabels)}`; + const series = streams.map((stream, index) => { + const colorIndex = index % colors.length; + stream.graphSeries.setColor(colors[colorIndex]); + stream.graphSeries.alias = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels), commonLabelsAlias); + return stream.graphSeries; + }); + const sortedEntries = _.chain(combinedEntries) .sortBy('timestamp') .reverse() .slice(0, limit || combinedEntries.length) .value(); - return { rows: sortedEntries }; + + meta.push({ + label: 'Limit', + value: `${limit} (${sortedEntries.length} returned)`, + }); + + return { meta, series, rows: sortedEntries }; +} + +export function processStream(stream: LogsStream, limit?: number): LogsStream { + const sortedEntries: any[] = _.chain(stream.entries) + .map(entry => processEntry(entry, stream)) + .sortBy('timestamp') + .reverse() + .slice(0, limit || stream.entries.length) + .value(); + + // Build graph data + let previousTime; + const datapoints = sortedEntries.reduce((acc, entry, index) => { + // Bucket to nearest minute + const time = Math.round(entry.timeJs / 1000 / 60) * 1000 * 60; + // Entry for time + if (time === previousTime) { + acc[acc.length - 1][0]++; + } else { + acc.push([1, time]); + previousTime = time; + } + return acc; + }, []); + const graphSeries = new TimeSeries({ datapoints, alias: stream.labels }); + + return { + ...stream, + graphSeries, + entries: sortedEntries, + parsedLabels: parseLabels(stream.labels), + }; } diff --git a/public/sass/components/_gf-form.scss b/public/sass/components/_gf-form.scss index 0de386f3f68..6d83fc6cf0b 100644 --- a/public/sass/components/_gf-form.scss +++ b/public/sass/components/_gf-form.scss @@ -116,6 +116,11 @@ $input-border: 1px solid $input-border-color; color: $critical; } + &--small { + padding: ($input-padding-y / 2) ($input-padding-x / 2); + font-size: $font-size-xs; + } + &:disabled { color: $text-color-weak; } diff --git a/public/sass/components/_switch.scss b/public/sass/components/_switch.scss index 6eb01ecc32d..c368d8ead67 100644 --- a/public/sass/components/_switch.scss +++ b/public/sass/components/_switch.scss @@ -41,7 +41,6 @@ bottom: 0; right: 0; color: #fff; - font-size: $font-size-sm; text-align: center; font-size: 150%; display: flex; @@ -91,6 +90,20 @@ transform: rotateY(0); } + &--small { + max-width: 2rem; + min-width: 1.5rem; + + input + label { + height: 25px; + } + + input + label::before, + input + label::after { + font-size: $font-size-sm; + } + } + &--table-cell { margin-bottom: 0; margin-right: 0; diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index b70b058879c..70b1901fb50 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -214,7 +214,42 @@ display: grid; grid-column-gap: 1rem; grid-row-gap: 0.1rem; - grid-template-columns: 4px minmax(100px, max-content) 1fr; + grid-template-columns: 4px minmax(100px, max-content) minmax(100px, 25%) 1fr; + font-family: $font-family-monospace; + font-size: 12px; + } + + .logs-controls { + display: flex; + + > * { + margin-right: 1em; + } + } + + .logs-options, + .logs-graph { + margin-bottom: $panel-margin; + } + + .logs-meta { + flex: 1; + color: $text-color-weak; + padding: 2px 0; + } + + .logs-meta-item { + display: inline-block; + margin-right: 1em; + } + + .logs-meta-item__label { + margin-right: 0.5em; + font-size: 0.9em; + font-weight: 500; + } + + .logs-meta-item__value { font-family: $font-family-monospace; }