+ ): this {
+ return this.addCustomEditor({
+ ...config,
+ id: config.path,
+ editor: standardEditorsRegistry.get('dashboard-uid').editor as any, // added at runtime
+ });
+ }
}
diff --git a/packages/grafana-e2e/src/flows/importDashboard.ts b/packages/grafana-e2e/src/flows/importDashboard.ts
index 81dc03c6d36..abf47860900 100644
--- a/packages/grafana-e2e/src/flows/importDashboard.ts
+++ b/packages/grafana-e2e/src/flows/importDashboard.ts
@@ -7,7 +7,7 @@ type Panel = {
[key: string]: unknown;
};
-type Dashboard = { title: string; panels: Panel[]; [key: string]: unknown };
+type Dashboard = { title: string; panels: Panel[]; uid: string; [key: string]: unknown };
/**
* Smoke test a datasource by quickly importing a test dashboard for it
@@ -17,10 +17,13 @@ export const importDashboard = (dashboardToImport: Dashboard) => {
e2e().visit(fromBaseUrl('/dashboard/import'));
// Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster
- e2e.components.DashboardImportPage.textarea().click({ force: true }).invoke('val', JSON.stringify(dashboardToImport));
- e2e.components.DashboardImportPage.submit().click({ force: true });
- e2e.components.ImportDashboardForm.name().click({ force: true }).clear().type(dashboardToImport.title);
- e2e.components.ImportDashboardForm.submit().click({ force: true });
+ e2e.components.DashboardImportPage.textarea()
+ .should('be.visible')
+ .click()
+ .invoke('val', JSON.stringify(dashboardToImport));
+ e2e.components.DashboardImportPage.submit().should('be.visible').click();
+ e2e.components.ImportDashboardForm.name().should('be.visible').click().clear().type(dashboardToImport.title);
+ e2e.components.ImportDashboardForm.submit().should('be.visible').click();
e2e().wait(3000);
// save the newly imported dashboard to context so it'll get properly deleted later
@@ -35,21 +38,23 @@ export const importDashboard = (dashboardToImport: Dashboard) => {
addedDashboards: [...addedDashboards, { title: dashboardToImport.title, uid }],
});
});
+
+ expect(dashboardToImport.uid).to.equal(uid);
});
// inspect first panel and verify data has been processed for it
- e2e.components.Panels.Panel.title(dashboardToImport.panels[0].title).click({ force: true });
- e2e.components.Panels.Panel.headerItems('Inspect').click({ force: true });
- e2e.components.Tab.title('JSON').click({ force: true });
+ e2e.components.Panels.Panel.title(dashboardToImport.panels[0].title).should('be.visible').click();
+ e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click();
+ e2e.components.Tab.title('JSON').should('be.visible').click();
e2e().wait(3000);
- e2e.components.PanelInspector.Json.content().contains('Panel JSON').click({ force: true });
+ e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click();
e2e().wait(3000);
- e2e.components.Select.option().contains('Data').click({ force: true });
+ e2e.components.Select.option().should('be.visible').contains('Data').click();
e2e().wait(3000);
// ensures that panel has loaded without knowingly hitting an error
// note: this does not prove that data came back as we expected it,
// it could get `state: Done` for no data for example
// but it ensures we didn't hit a 401 or 500 or something like that
- e2e.components.CodeEditor.container().contains('"state": "Done"');
+ e2e.components.CodeEditor.container().should('be.visible').contains('"state": "Done"');
};
diff --git a/packages/grafana-runtime/src/components/PanelRenderer.tsx b/packages/grafana-runtime/src/components/PanelRenderer.tsx
index 1d178836483..23c140408a4 100644
--- a/packages/grafana-runtime/src/components/PanelRenderer.tsx
+++ b/packages/grafana-runtime/src/components/PanelRenderer.tsx
@@ -13,10 +13,11 @@ export interface PanelRendererProps;
options?: P;
onOptionsChange?: (options: P) => void;
onChangeTimeRange?: (timeRange: AbsoluteTimeRange) => void;
+ fieldConfig?: FieldConfigSource;
+ onFieldConfigChange?: (config: FieldConfigSource) => void;
timeZone?: string;
width: number;
height: number;
diff --git a/packages/grafana-runtime/src/config.ts b/packages/grafana-runtime/src/config.ts
index d1a23c057fc..ddd10809874 100644
--- a/packages/grafana-runtime/src/config.ts
+++ b/packages/grafana-runtime/src/config.ts
@@ -66,6 +66,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
tempoServiceGraph: false,
tempoSearch: false,
prometheusMonaco: false,
+ newNavigation: false,
};
licenseInfo: LicenseInfo = {} as LicenseInfo;
rendererAvailable = false;
diff --git a/packages/grafana-schema/src/schema/graph.gen.ts b/packages/grafana-schema/src/schema/graph.gen.ts
index 89260ac9289..081fb369005 100644
--- a/packages/grafana-schema/src/schema/graph.gen.ts
+++ b/packages/grafana-schema/src/schema/graph.gen.ts
@@ -1,6 +1,6 @@
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// NOTE: This file will be auto generated from models.cue
-// It is currenty hand written but will serve as the target for cuetsy
+// It is currently hand written but will serve as the target for cuetsy
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
export enum AxisPlacement {
diff --git a/packages/grafana-schema/src/scuemata/dashboard/dashboard.cue b/packages/grafana-schema/src/scuemata/dashboard/dashboard.cue
index 1aefbace0ca..15a0c4b7fd5 100644
--- a/packages/grafana-schema/src/scuemata/dashboard/dashboard.cue
+++ b/packages/grafana-schema/src/scuemata/dashboard/dashboard.cue
@@ -1,10 +1,14 @@
package dashboard
-import "github.com/grafana/grafana/cue/scuemata"
+import (
+ "list"
+
+ "github.com/grafana/grafana/cue/scuemata"
+)
Family: scuemata.#Family & {
lineages: [
- [
+ [
{ // 0.0
// Unique numeric identifier for the dashboard.
// TODO must isolate or remove identifiers local to a Grafana instance...?
@@ -131,9 +135,7 @@ Family: scuemata.#Family & {
// Dashboard panels. Panels are canonically defined inline
// because they share a version timeline with the dashboard
- // schema; they do not vary independently. We create a separate,
- // synthetic Family to represent them in Go, for ease of generating
- // e.g. JSON Schema.
+ // schema; they do not evolve independently.
#Panel: {
// The panel plugin type id.
type: !=""
@@ -155,7 +157,8 @@ Family: scuemata.#Family & {
// _pv: { maj: int, min: int }
// The major and minor versions of the panel plugin for this schema.
// TODO 2-tuple list instead of struct?
- panelSchema?: { maj: number, min: number }
+ // panelSchema?: { maj: number, min: number }
+ panelSchema?: [number, number]
// TODO docs
targets?: [...#Target]
@@ -218,9 +221,8 @@ Family: scuemata.#Family & {
// TODO tighter constraint
timeShift?: string
- // The allowable options are specified by the panel plugin's
- // schema.
- // FIXME same conundrum as with the closed validation for fieldConfig.
+ // options is specified by the PanelOptions field in panel
+ // plugin schemas.
options: {}
fieldConfig: {
@@ -283,16 +285,8 @@ Family: scuemata.#Family & {
// Alternative to empty string
noValue?: string
- // TODO conundrum: marking this struct as open would
- // - i think - preclude closed validation of
- // plugin-defined config bits. But, marking it
- // closed makes it impossible to use just this
- // schema (the "base" variant) to validate the base
- // components of a dashboard.
- //
- // Can always exist. Valid fields within this are
- // defined by the panel plugin - that's the
- // PanelFieldConfig that comes from the plugin.
+ // custom is specified by the PanelFieldConfig field
+ // in panel plugin schemas.
custom?: {}
}
overrides: [...{
@@ -306,7 +300,29 @@ Family: scuemata.#Family & {
}]
}]
}
+ // Embed the disjunction of all injected panel schema, if any were injected.
+ if len(compose._panelSchemas) > 0 {
+ or(compose._panelSchemas) // TODO try to stick graph in here
+ }
+
+ // Make the plugin-composed subtrees open if the panel is
+ // of unknown types. This is important in every possible case:
+ // - Base (this file only): no real dashboard json
+ // containing any panels would ever validate
+ // - Dist (this file + core plugin schema): dashboard json containing
+ // panels with any third-party panel plugins would fail to validate,
+ // as well as any core plugins lacking a models.cue. The latter case
+ // is not normally expected, but this is not the appropriate place
+ // to enforce the invariant, anyway.
+ // - Instance (this file + core + third-party plugin schema): dashboard
+ // json containing panels with a third-party plugin that exists but
+ // is not currently installed would fail to validate.
+ if !list.Contains(compose._panelTypes, type) {
+ options: {...}
+ fieldConfig: defaults: custom: {...}
+ }
}
+
// Row panel
#RowPanel: {
type: "row"
@@ -329,86 +345,84 @@ Family: scuemata.#Family & {
static?: bool
}
id: number
- panels: [...#Panel | #GraphPanel]
+ panels: [...(#Panel | #GraphPanel)]
}
// Support for legacy graph panels.
#GraphPanel: {
...
type: "graph"
thresholds: [...{...}]
- timeRegions: [...{...}]
- // FIXME this one is quite complicated, as it duplicates the #Panel object's own structure (...?)
+ timeRegions?: [...{...}]
seriesOverrides: [...{...}]
-
- // TODO docs
- // TODO tighter constraint
aliasColors?: [string]: string
-
- // TODO docs
bars: bool | *false
- // TODO docs
dashes: bool | *false
- // TODO docs
dashLength: number | *10
- // TODO docs
- // TODO tighter constraint
fill?: number
- // TODO docs
- // TODO tighter constraint
fillGradient?: number
-
- // TODO docs
hiddenSeries: bool | *false
-
- // FIXME idk where this comes from, leaving it very open and very wrong for now
legend: {...}
-
- // TODO docs
- // TODO tighter constraint
lines: bool | *false
- // TODO docs
linewidth?: number
- // TODO docs
nullPointMode: *"null" | "connected" | "null as zero"
- // TODO docs
percentage: bool | *false
- // TODO docs
points: bool | *false
- // TODO docs
- // FIXME this is the kind of case that makes
- // optional/non-default tricky: it's optional because it
- // only makes sense when points is true (right?), but if it
- // is, then there actually is a default value. Easier way to
- // represent this would be to wrap up this handling into a
- // struct
pointradius?: number
- // TODO docs
- // TODO tighter constraint
renderer: string
- // TODO docs
spaceLength: number | *10
- // TODO docs
stack: bool | *false
- // TODO docs
steppedLine: bool | *false
- // TODO docs
tooltip?: {
- // TODO docs
shared?: bool
- // TODO docs
sort: number | *0
- // TODO docs
- // FIXME literally no idea if these values are sane
value_type: *"individual" | "cumulative"
}
-
}
}
]
]
-}
+ compose: {
+ // Scuemata families for all panel types that should be composed into the
+ // dashboard schema.
+ Panel: [string]: scuemata.#PanelFamily
-#Latest: {
- #Dashboard: Family.latest
- #Panel: Family.latest._Panel
-}
+ // _panelTypes: [for typ, _ in Panel {typ}]
+ _panelTypes: [for typ, _ in Panel {typ}, "graph", "row"]
+ _panelSchemas: [for typ, scue in Panel {
+ for lv, lin in scue.lineages {
+ for sv, sch in lin {
+ (_mapPanel & {arg: {
+ type: typ
+ v: [lv, sv] // TODO add optionality for exact, at least, at most, any
+ model: sch // TODO Does this need to be close()d?
+ }}).out
+ }
+ }
+ }, { type: string }]
+ _mapPanel: {
+ arg: {
+ type: string & !=""
+ v: [number, number]
+ model: {...}
+ }
+ // Until CUE introduces the must() constraint, we have to enforce
+ // that the input model is as expected by checking for unification
+ // in this hidden property (see https://github.com/cue-lang/cue/issues/575).
+ // If we unified arg.model with the scuemata.#PanelSchema
+ // meta-schema directly, the struct openness (PanelOptions: {...})
+ // would be applied to the actual schema instance in the arg. Here,
+ // where we're actually putting those in the dashboard schema, want
+ // those to be closed, or at least preserve closed-ness.
+ _checkSchema: scuemata.#PanelSchema & arg.model
+ out: {
+ type: arg.type
+ panelSchema: arg.v // TODO add optionality for exact, at least, at most, any
+ options: arg.model.PanelOptions
+ fieldConfig: defaults: custom: {}
+ if arg.model.PanelFieldConfig != _|_ {
+ fieldConfig: defaults: custom: arg.model.PanelFieldConfig
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/packages/grafana-toolkit/package.json b/packages/grafana-toolkit/package.json
index 16ed527d98e..c9981e4db3a 100644
--- a/packages/grafana-toolkit/package.json
+++ b/packages/grafana-toolkit/package.json
@@ -47,7 +47,7 @@
"@types/webpack": "4.41.7",
"@typescript-eslint/eslint-plugin": "4.28.0",
"@typescript-eslint/parser": "4.28.0",
- "axios": "0.21.1",
+ "axios": "0.21.2",
"babel-jest": "26.6.3",
"babel-loader": "8.2.2",
"babel-plugin-angularjs-annotate": "0.10.0",
diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json
index 92f80dc8a7f..d8dd362d0e8 100644
--- a/packages/grafana-ui/package.json
+++ b/packages/grafana-ui/package.json
@@ -38,7 +38,7 @@
"@grafana/schema": "8.2.0-pre",
"@grafana/slate-react": "0.22.10-grafana",
"@grafana/tsconfig": "^1.0.0-rc1",
- "@monaco-editor/react": "4.1.1",
+ "@monaco-editor/react": "4.2.2",
"@popperjs/core": "2.5.4",
"@sentry/browser": "5.25.0",
"ansicolor": "1.1.95",
@@ -50,7 +50,7 @@
"jquery": "3.5.1",
"lodash": "4.17.21",
"moment": "2.29.1",
- "monaco-editor": "0.21.2",
+ "monaco-editor": "0.27.0",
"papaparse": "5.3.0",
"rc-cascader": "1.5.0",
"rc-drawer": "4.4.0",
diff --git a/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx b/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx
index a3984f2faad..73c78e5d172 100644
--- a/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx
+++ b/packages/grafana-ui/src/components/DataSourceSettings/DataSourceHttpSettings.tsx
@@ -163,7 +163,7 @@ export const DataSourceHttpSettings: React.FC = (props) => {
width={13}
tooltip="Grafana proxy deletes forwarded cookies by default. Specify cookies by name that should be forwarded to the data source."
>
- Whitelisted Cookies
+ Allowed cookies
{
+ let nextValue = value ?? '';
if (e.hasOwnProperty('key')) {
// handling keyboard event
const evt = e as React.KeyboardEvent;
if (evt.key === 'Enter' && !item.settings?.useTextarea) {
- onChange(evt.currentTarget.value.trim() === '' ? undefined : evt.currentTarget.value);
+ nextValue = evt.currentTarget.value.trim();
}
} else {
// handling form event
const evt = e as React.FormEvent;
- onChange(evt.currentTarget.value.trim() === '' ? undefined : evt.currentTarget.value);
+ nextValue = evt.currentTarget.value.trim();
}
+ if (nextValue === value) {
+ return; // no change
+ }
+ onChange(nextValue === '' ? undefined : nextValue);
},
- [item.settings?.useTextarea, onChange]
+ [value, item.settings?.useTextarea, onChange]
);
return (
diff --git a/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts b/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts
index 8c10709b1e4..bfe31ef4ef4 100644
--- a/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts
+++ b/packages/grafana-ui/src/components/PanelChrome/PanelContext.ts
@@ -1,4 +1,4 @@
-import { EventBusSrv, EventBus, DashboardCursorSync, AnnotationEventUIModel } from '@grafana/data';
+import { AnnotationEventUIModel, DashboardCursorSync, EventBus, EventBusSrv, SplitOpen } from '@grafana/data';
import React from 'react';
import { SeriesVisibilityChangeMode } from '.';
@@ -22,6 +22,11 @@ export interface PanelContext {
onAnnotationCreate?: (annotation: AnnotationEventUIModel) => void;
onAnnotationUpdate?: (annotation: AnnotationEventUIModel) => void;
onAnnotationDelete?: (id: string) => void;
+ /**
+ * onSplitOpen is used in Explore to open the split view. It can be used in panels which has intercations and used in Explore as well.
+ * For example TimeSeries panel.
+ */
+ onSplitOpen?: SplitOpen;
}
export const PanelContextRoot = React.createContext({
diff --git a/packages/grafana-ui/src/components/Select/SelectMenu.tsx b/packages/grafana-ui/src/components/Select/SelectMenu.tsx
index 008d871ba93..628c8660365 100644
--- a/packages/grafana-ui/src/components/Select/SelectMenu.tsx
+++ b/packages/grafana-ui/src/components/Select/SelectMenu.tsx
@@ -4,6 +4,8 @@ import { getSelectStyles } from './getSelectStyles';
import { cx } from '@emotion/css';
import { SelectableValue } from '@grafana/data';
import { CustomScrollbar } from '../CustomScrollbar/CustomScrollbar';
+import { Icon } from '../Icon/Icon';
+import { IconName } from '../../types';
interface SelectMenuProps {
maxHeight: number;
@@ -49,6 +51,7 @@ export const SelectMenuOptions = React.forwardRef
+ {data.icon && }
{data.imgUrl && 
}
{renderOptionLabel ? renderOptionLabel(data) : children}
diff --git a/packages/grafana-ui/src/components/Select/getSelectStyles.ts b/packages/grafana-ui/src/components/Select/getSelectStyles.ts
index bc60888cc3f..8821b1cc74e 100644
--- a/packages/grafana-ui/src/components/Select/getSelectStyles.ts
+++ b/packages/grafana-ui/src/components/Select/getSelectStyles.ts
@@ -27,6 +27,9 @@ export const getSelectStyles = stylesFactory((theme: GrafanaTheme2) => {
background: ${theme.colors.action.hover};
}
`,
+ optionIcon: css`
+ margin-right: ${theme.spacing(1)};
+ `,
optionImage: css`
label: grafana-select-option-image;
width: 16px;
diff --git a/packages/grafana-ui/src/components/Table/HeaderRow.tsx b/packages/grafana-ui/src/components/Table/HeaderRow.tsx
index 15339c470bb..1b513d5baf9 100644
--- a/packages/grafana-ui/src/components/Table/HeaderRow.tsx
+++ b/packages/grafana-ui/src/components/Table/HeaderRow.tsx
@@ -6,14 +6,16 @@ import { getTableStyles, TableStyles } from './styles';
import { useStyles2 } from '../../themes';
import { Filter } from './Filter';
import { Icon } from '../Icon/Icon';
+import { getFieldTypeIcon } from '../../types';
export interface HeaderRowProps {
headerGroups: HeaderGroup[];
data: DataFrame;
+ showTypeIcons?: boolean;
}
export const HeaderRow = (props: HeaderRowProps) => {
- const { headerGroups, data } = props;
+ const { headerGroups, data, showTypeIcons } = props;
const e2eSelectorsTable = selectors.components.Panels.Visualization.Table;
const tableStyles = useStyles2(getTableStyles);
@@ -30,7 +32,7 @@ export const HeaderRow = (props: HeaderRowProps) => {
role="row"
>
{headerGroup.headers.map((column: Column, index: number) =>
- renderHeaderCell(column, tableStyles, data.fields[index])
+ renderHeaderCell(column, tableStyles, data.fields[index], showTypeIcons)
)}
);
@@ -39,7 +41,7 @@ export const HeaderRow = (props: HeaderRowProps) => {
);
};
-function renderHeaderCell(column: any, tableStyles: TableStyles, field?: Field) {
+function renderHeaderCell(column: any, tableStyles: TableStyles, field?: Field, showTypeIcons?: boolean) {
const headerProps = column.getHeaderProps();
if (column.canResize) {
@@ -58,6 +60,9 @@ function renderHeaderCell(column: any, tableStyles: TableStyles, field?: Field)
className={tableStyles.headerCellLabel}
title={column.render('Header')}
>
+ {showTypeIcons && (
+
+ )}
{column.render('Header')}
{column.isSorted && (column.isSortedDesc ?
:
)}
diff --git a/packages/grafana-ui/src/components/Table/JSONViewCell.tsx b/packages/grafana-ui/src/components/Table/JSONViewCell.tsx
index 15d5636b118..d163a151672 100644
--- a/packages/grafana-ui/src/components/Table/JSONViewCell.tsx
+++ b/packages/grafana-ui/src/components/Table/JSONViewCell.tsx
@@ -58,7 +58,7 @@ function getStyles(theme: GrafanaTheme2) {
padding: ${theme.spacing(0.5)};
`,
json: css`
- max-width: fit-content;
+ width: fit-content;
max-height: 70vh;
overflow-y: auto;
`,
diff --git a/packages/grafana-ui/src/components/Table/Table.tsx b/packages/grafana-ui/src/components/Table/Table.tsx
index 2def06e4e7a..cc6772b9047 100644
--- a/packages/grafana-ui/src/components/Table/Table.tsx
+++ b/packages/grafana-ui/src/components/Table/Table.tsx
@@ -38,6 +38,7 @@ export interface Props {
/** Minimal column width specified in pixels */
columnMinWidth?: number;
noHeader?: boolean;
+ showTypeIcons?: boolean;
resizable?: boolean;
initialSortBy?: TableSortByFieldState[];
onColumnResize?: TableColumnResizeActionCallback;
@@ -124,6 +125,7 @@ export const Table: FC
= memo((props: Props) => {
resizable = true,
initialSortBy,
footerValues,
+ showTypeIcons,
} = props;
const tableStyles = useStyles2(getTableStyles);
@@ -204,7 +206,7 @@ export const Table: FC = memo((props: Props) => {
- {!noHeader &&
}
+ {!noHeader &&
}
{rows.length > 0 ? (
{
nextValue = steps[steps.length - 1].value + 10;
}
- const color = colors.filter((c) => !steps.some((t) => t.color === c))[1];
+ let color = colors.filter((c) => !steps.some((t) => t.color === c))[1];
+ if (!color) {
+ // Default color when all colors are used
+ color = '#CCCCCC';
+ }
const add = {
value: nextValue,
@@ -154,15 +158,13 @@ export class ThresholdsEditor extends PureComponent {
value={'Base'}
disabled
prefix={
- threshold.color && (
-
- this.onChangeThresholdColor(threshold, color)}
- enableNamedColors={true}
- />
-
- )
+
+ this.onChangeThresholdColor(threshold, color)}
+ enableNamedColors={true}
+ />
+
}
/>
);
@@ -179,15 +181,13 @@ export class ThresholdsEditor extends PureComponent {
onBlur={this.onBlur}
prefix={
- {threshold.color && (
-
- this.onChangeThresholdColor(threshold, color)}
- enableNamedColors={true}
- />
-
- )}
+
+ this.onChangeThresholdColor(threshold, color)}
+ enableNamedColors={true}
+ />
+
{isPercent &&
%
}
-
- {this.props.children}
-
+
+ {this.props.children}
+
{children}
;
};
diff --git a/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx b/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx
index 5fe92461307..923708e218d 100644
--- a/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx
+++ b/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin.tsx
@@ -1,7 +1,3 @@
-import React, { useEffect, useLayoutEffect, useState } from 'react';
-import { Portal } from '../../Portal/Portal';
-import { usePlotContext } from '../context';
-import { TooltipDisplayMode } from '@grafana/schema';
import {
CartesianCoords2D,
DashboardCursorSync,
@@ -13,11 +9,15 @@ import {
getFieldDisplayName,
TimeZone,
} from '@grafana/data';
+import { TooltipDisplayMode } from '@grafana/schema';
+import React, { useEffect, useLayoutEffect, useState } from 'react';
+import { useMountedState } from 'react-use';
+import uPlot from 'uplot';
+import { useTheme2 } from '../../../themes/ThemeContext';
+import { Portal } from '../../Portal/Portal';
import { SeriesTable, SeriesTableRowProps, VizTooltipContainer } from '../../VizTooltip';
import { UPlotConfigBuilder } from '../config/UPlotConfigBuilder';
import { findMidPointYPosition, pluginLog } from '../utils';
-import { useTheme2 } from '../../../themes/ThemeContext';
-import uPlot from 'uplot';
interface TooltipPluginProps {
timeZone: TimeZone;
@@ -44,13 +44,12 @@ export const TooltipPlugin: React.FC = ({
...otherProps
}) => {
const theme = useTheme2();
- const plotCtx = usePlotContext();
const [focusedSeriesIdx, setFocusedSeriesIdx] = useState(null);
const [focusedPointIdx, setFocusedPointIdx] = useState(null);
const [focusedPointIdxs, setFocusedPointIdxs] = useState>([]);
const [coords, setCoords] = useState(null);
- const plotInstance = plotCtx.plot;
const [isActive, setIsActive] = useState(false);
+ const isMounted = useMountedState();
const pluginId = `TooltipPlugin`;
@@ -59,42 +58,46 @@ export const TooltipPlugin: React.FC = ({
pluginLog(pluginId, true, `Focused series: ${focusedSeriesIdx}, focused point: ${focusedPointIdx}`);
}, [focusedPointIdx, focusedSeriesIdx]);
- useEffect(() => {
+ // Add uPlot hooks to the config, or re-add when the config changed
+ useLayoutEffect(() => {
+ let plotInstance: uPlot | undefined = undefined;
+ let bbox: DOMRect | undefined = undefined;
+
const plotMouseLeave = () => {
+ if (!isMounted()) {
+ return;
+ }
setCoords(null);
setIsActive(false);
- if (plotCtx.plot) {
- plotCtx.plot.root.classList.remove('plot-active');
- }
+ plotInstance?.root.classList.remove('plot-active');
};
const plotMouseEnter = () => {
+ if (!isMounted()) {
+ return;
+ }
setIsActive(true);
- if (plotCtx.plot) {
- plotCtx.plot.root.classList.add('plot-active');
- }
+ plotInstance?.root.classList.add('plot-active');
};
- if (plotCtx && plotCtx.plot) {
- plotCtx.plot.over.addEventListener('mouseleave', plotMouseLeave);
- plotCtx.plot.over.addEventListener('mouseenter', plotMouseEnter);
+ // cache uPlot plotting area bounding box
+ config.addHook('syncRect', (u, rect) => {
+ bbox = rect;
+ });
+
+ config.addHook('init', (u) => {
+ plotInstance = u;
+
+ u.over.addEventListener('mouseleave', plotMouseLeave);
+ u.over.addEventListener('mouseenter', plotMouseEnter);
+
if (sync === DashboardCursorSync.Crosshair) {
- plotCtx.plot.root.classList.add('shared-crosshair');
+ u.root.classList.add('shared-crosshair');
}
- }
+ });
- return () => {
- setCoords(null);
- if (plotCtx && plotCtx.plot) {
- plotCtx.plot.over.removeEventListener('mouseleave', plotMouseLeave);
- plotCtx.plot.over.removeEventListener('mouseenter', plotMouseEnter);
- }
- };
- }, [plotCtx.plot?.root]);
-
- // Add uPlot hooks to the config, or re-add when the config changed
- useLayoutEffect(() => {
const tooltipInterpolator = config.getTooltipInterpolator();
+
if (tooltipInterpolator) {
// Custom toolitp positioning
config.addHook('setCursor', (u) => {
@@ -107,7 +110,6 @@ export const TooltipPlugin: React.FC = ({
return;
}
- const bbox = plotCtx.getCanvasBoundingBox();
if (!bbox) {
return;
}
@@ -122,14 +124,16 @@ export const TooltipPlugin: React.FC = ({
});
} else {
config.addHook('setLegend', (u) => {
+ if (!isMounted()) {
+ return;
+ }
setFocusedPointIdx(u.legend.idx!);
setFocusedPointIdxs(u.legend.idxs!.slice());
});
// default series/datapoint idx retireval
config.addHook('setCursor', (u) => {
- const bbox = plotCtx.getCanvasBoundingBox();
- if (!bbox) {
+ if (!bbox || !isMounted()) {
return;
}
@@ -142,12 +146,23 @@ export const TooltipPlugin: React.FC = ({
});
config.addHook('setSeries', (_, idx) => {
+ if (!isMounted()) {
+ return;
+ }
setFocusedSeriesIdx(idx);
});
}
- }, [plotCtx, config]);
- if (!plotInstance || focusedPointIdx === null || (!isActive && sync === DashboardCursorSync.Crosshair)) {
+ return () => {
+ setCoords(null);
+ if (plotInstance) {
+ plotInstance.over.removeEventListener('mouseleave', plotMouseLeave);
+ plotInstance.over.removeEventListener('mouseenter', plotMouseEnter);
+ }
+ };
+ }, [config, setCoords, setIsActive, setFocusedPointIdx, setFocusedPointIdxs]);
+
+ if (focusedPointIdx === null || (!isActive && sync === DashboardCursorSync.Crosshair)) {
return null;
}
@@ -189,10 +204,10 @@ export const TooltipPlugin: React.FC = ({
if (mode === TooltipDisplayMode.Multi) {
let series: SeriesTableRowProps[] = [];
- const plotSeries = plotInstance.series;
+ const frame = otherProps.data;
+ const fields = frame.fields;
- for (let i = 0; i < plotSeries.length; i++) {
- const frame = otherProps.data;
+ for (let i = 0; i < fields.length; i++) {
const field = frame.fields[i];
if (
!field ||
diff --git a/packages/grafana-ui/src/slate-plugins/clipboard.ts b/packages/grafana-ui/src/slate-plugins/clipboard.ts
index 5137bcc09e9..ee558b288b4 100644
--- a/packages/grafana-ui/src/slate-plugins/clipboard.ts
+++ b/packages/grafana-ui/src/slate-plugins/clipboard.ts
@@ -10,6 +10,11 @@ const getCopiedText = (textBlocks: string[], startOffset: number, endOffset: num
return textBlocks.join('\n').slice(startOffset, excludingLastLineLength + endOffset);
};
+// Remove unicode special symbol - byte order mark (BOM), U+FEFF.
+const removeBom = (str: string | undefined): string | undefined => {
+ return str?.replace(/[\uFEFF]/g, '');
+};
+
export function ClipboardPlugin(): Plugin {
const clipboardPlugin: Plugin = {
onCopy(event: Event, editor: CoreEditor, next: () => any) {
@@ -26,7 +31,7 @@ export function ClipboardPlugin(): Plugin {
.toArray()
.map((block) => block.text);
- const copiedText = getCopiedText(selectedBlocks, startOffset, endOffset);
+ const copiedText = removeBom(getCopiedText(selectedBlocks, startOffset, endOffset));
if (copiedText && clipEvent.clipboardData) {
clipEvent.clipboardData.setData('Text', copiedText);
}
@@ -38,10 +43,10 @@ export function ClipboardPlugin(): Plugin {
const clipEvent = event as ClipboardEvent;
clipEvent.preventDefault();
if (clipEvent.clipboardData) {
- const pastedValue = clipEvent.clipboardData.getData('Text');
- const lines = pastedValue.split('\n');
+ const pastedValue = removeBom(clipEvent.clipboardData.getData('Text'));
+ const lines = pastedValue?.split('\n');
- if (lines.length) {
+ if (lines && lines.length) {
editor.insertText(lines[0]);
for (const line of lines.slice(1)) {
editor.splitBlock().insertText(line);
diff --git a/packages/grafana-ui/src/types/icon.ts b/packages/grafana-ui/src/types/icon.ts
index b2bb10fafd8..93f5e4b3c21 100644
--- a/packages/grafana-ui/src/types/icon.ts
+++ b/packages/grafana-ui/src/types/icon.ts
@@ -1,3 +1,4 @@
+import { Field, FieldType } from '@grafana/data';
import { ComponentSize } from './size';
export type IconType = 'mono' | 'default';
export type IconSize = ComponentSize | 'xl' | 'xxl' | 'xxxl';
@@ -68,6 +69,7 @@ export const getAvailableIcons = () =>
'file-copy-alt',
'filter',
'folder',
+ 'font',
'fire',
'folder-open',
'folder-plus',
@@ -140,6 +142,7 @@ export const getAvailableIcons = () =>
'table',
'tag-alt',
'times',
+ 'toggle-on',
'trash-alt',
'unlock',
'upload',
@@ -152,3 +155,24 @@ export const getAvailableIcons = () =>
type BrandIconNames = 'google' | 'microsoft' | 'github' | 'gitlab' | 'okta';
export type IconName = ReturnType[number] | BrandIconNames;
+
+/** Get the icon for a given field type */
+export function getFieldTypeIcon(field?: Field): IconName {
+ if (field) {
+ switch (field.type) {
+ case FieldType.time:
+ return 'clock-nine';
+ case FieldType.string:
+ return 'font';
+ case FieldType.number:
+ return 'calculator-alt';
+ case FieldType.boolean:
+ return 'toggle-on';
+ case FieldType.trace:
+ return 'info-circle';
+ case FieldType.other:
+ return 'brackets-curly';
+ }
+ }
+ return 'question-circle';
+}
diff --git a/packages/grafana-ui/src/utils/standardEditors.tsx b/packages/grafana-ui/src/utils/standardEditors.tsx
index 4275cd55113..5f094f7fecc 100644
--- a/packages/grafana-ui/src/utils/standardEditors.tsx
+++ b/packages/grafana-ui/src/utils/standardEditors.tsx
@@ -229,6 +229,8 @@ export const getStandardFieldConfigs = () => {
/**
* Returns collection of standard option editors definitions
+ *
+ * @internal
*/
export const getStandardOptionEditors = () => {
const number: StandardEditorsRegistryItem = {
diff --git a/packages/jaeger-ui-components/src/TracePageHeader/TracePageHeader.tsx b/packages/jaeger-ui-components/src/TracePageHeader/TracePageHeader.tsx
index 224d13d6f88..f1c69988e20 100644
--- a/packages/jaeger-ui-components/src/TracePageHeader/TracePageHeader.tsx
+++ b/packages/jaeger-ui-components/src/TracePageHeader/TracePageHeader.tsx
@@ -26,12 +26,13 @@ import TraceName from '../common/TraceName';
import { getTraceName } from '../model/trace-viewer';
import { TNil } from '../types';
import { Trace } from '../types/trace';
-import { formatDatetime, formatDuration } from '../utils/date';
+import { formatDuration } from '../utils/date';
import { getTraceLinks } from '../model/link-patterns';
import ExternalLinks from '../common/ExternalLinks';
import { createStyle } from '../Theme';
import { uTxMuted } from '../uberUtilityStyles';
+import { dateTimeFormat, TimeZone } from '@grafana/data';
const getStyles = createStyle((theme: Theme) => {
return {
@@ -157,14 +158,16 @@ type TracePageHeaderEmbedProps = {
searchValue: string;
onSearchValueChange: (value: string) => void;
hideSearchButtons?: boolean;
+ timeZone: TimeZone;
};
export const HEADER_ITEMS = [
{
key: 'timestamp',
label: 'Trace Start',
- renderer(trace: Trace, styles: ReturnType) {
- const dateStr = formatDatetime(trace.startTime);
+ renderer(trace: Trace, timeZone: TimeZone, styles: ReturnType) {
+ // Convert date from micro to milli seconds
+ const dateStr = dateTimeFormat(trace.startTime / 1000, { timeZone });
const match = dateStr.match(/^(.+)(:\d\d\.\d+)$/);
return match ? (
@@ -219,6 +222,7 @@ export default function TracePageHeader(props: TracePageHeaderEmbedProps) {
searchValue,
onSearchValueChange,
hideSearchButtons,
+ timeZone,
} = props;
const styles = getStyles(useTheme());
@@ -238,7 +242,7 @@ export default function TracePageHeader(props: TracePageHeaderEmbedProps) {
!slimView &&
HEADER_ITEMS.map((item) => {
const { renderer, ...rest } = item;
- return { ...rest, value: renderer(trace, styles) };
+ return { ...rest, value: renderer(trace, timeZone, styles) };
});
const title = (
diff --git a/packaging/deb/default/grafana-server b/packaging/deb/default/grafana-server
index eb77e62d774..cd0580aa4ff 100644
--- a/packaging/deb/default/grafana-server
+++ b/packaging/deb/default/grafana-server
@@ -21,4 +21,4 @@ PLUGINS_DIR=/var/lib/grafana/plugins
PROVISIONING_CFG_DIR=/etc/grafana/provisioning
# Only used on systemd systems
-PID_FILE_DIR=/var/run/grafana
+PID_FILE_DIR=/run/grafana
diff --git a/pkg/api/api.go b/pkg/api/api.go
index cbfb05c299c..140f243534f 100644
--- a/pkg/api/api.go
+++ b/pkg/api/api.go
@@ -72,9 +72,12 @@ func (hs *HTTPServer) registerRoutes() {
r.Get("/admin/orgs/edit/:id", reqGrafanaAdmin, hs.Index)
r.Get("/admin/stats", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionServerStatsRead)), hs.Index)
r.Get("/admin/ldap", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionLDAPStatusRead)), hs.Index)
-
r.Get("/styleguide", reqSignedIn, hs.Index)
+ r.Get("/live", reqGrafanaAdmin, hs.Index)
+ r.Get("/live/pipeline", reqGrafanaAdmin, hs.Index)
+ r.Get("/live/cloud", reqGrafanaAdmin, hs.Index)
+
r.Get("/plugins", reqSignedIn, hs.Index)
r.Get("/plugins/:id/", reqSignedIn, hs.Index)
r.Get("/plugins/:id/edit", reqSignedIn, hs.Index) // deprecated
@@ -133,7 +136,7 @@ func (hs *HTTPServer) registerRoutes() {
r.Get("/api/login/ping", quota("session"), routing.Wrap(hs.LoginAPIPing))
// expose plugin file system assets
- r.Get("/public/plugins/:pluginId/*", hs.GetPluginAssets)
+ r.Get("/public/plugins/:pluginId/*", hs.getPluginAssets)
// authed api
r.Group("/api", func(apiRoute routing.RouteRegister) {
@@ -266,7 +269,7 @@ func (hs *HTTPServer) registerRoutes() {
// Data sources
apiRoute.Group("/datasources", func(datasourceRoute routing.RouteRegister) {
- datasourceRoute.Get("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead)), routing.Wrap(hs.GetDataSources))
+ datasourceRoute.Get("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead, ScopeDatasourcesAll)), routing.Wrap(hs.GetDataSources))
datasourceRoute.Post("/", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesCreate)), quota("data_source"), bind(models.AddDataSourceCommand{}), routing.Wrap(AddDataSource))
datasourceRoute.Put("/:id", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesWrite, ScopeDatasourceID)), bind(models.UpdateDataSourceCommand{}), routing.Wrap(hs.UpdateDataSource))
datasourceRoute.Delete("/:id", authorize(reqOrgAdmin, ac.EvalPermission(ActionDatasourcesDelete, ScopeDatasourceID)), routing.Wrap(hs.DeleteDataSourceById))
@@ -370,8 +373,6 @@ func (hs *HTTPServer) registerRoutes() {
// metrics
apiRoute.Post("/tsdb/query", bind(dtos.MetricRequest{}), routing.Wrap(hs.QueryMetrics))
- apiRoute.Get("/tsdb/testdata/gensql", reqGrafanaAdmin, routing.Wrap(GenerateSQLTestData))
- apiRoute.Get("/tsdb/testdata/random-walk", routing.Wrap(hs.GetTestDataRandomWalk))
// DataSource w/ expressions
apiRoute.Post("/ds/query", bind(dtos.MetricRequest{}), routing.Wrap(hs.QueryMetricsV2))
@@ -423,7 +424,7 @@ func (hs *HTTPServer) registerRoutes() {
// the channel path is in the name
liveRoute.Post("/publish", bind(dtos.LivePublishCmd{}), routing.Wrap(hs.Live.HandleHTTPPublish))
- // POST influx line protocol
+ // POST influx line protocol.
liveRoute.Post("/push/:streamId", hs.LivePushGateway.Handle)
// List available streams and fields
@@ -431,6 +432,13 @@ func (hs *HTTPServer) registerRoutes() {
// Some channels may have info
liveRoute.Get("/info/*", routing.Wrap(hs.Live.HandleInfoHTTP))
+
+ if hs.Cfg.FeatureToggles["live-pipeline"] {
+ // POST Live data to be processed according to channel rules.
+ liveRoute.Post("/push/:streamId/:path", hs.LivePushGateway.HandlePath)
+ liveRoute.Get("/channel-rules", routing.Wrap(hs.Live.HandleChannelRulesListHTTP), reqOrgAdmin)
+ liveRoute.Get("/remote-write-backends", routing.Wrap(hs.Live.HandleRemoteWriteBackendsListHTTP), reqOrgAdmin)
+ }
})
// short urls
diff --git a/pkg/api/app_routes.go b/pkg/api/app_routes.go
index b30a757cd76..bc2ef26948f 100644
--- a/pkg/api/app_routes.go
+++ b/pkg/api/app_routes.go
@@ -4,6 +4,7 @@ import (
"crypto/tls"
"net"
"net/http"
+ "strings"
"time"
"github.com/grafana/grafana/pkg/api/pluginproxy"
@@ -47,7 +48,9 @@ func (hs *HTTPServer) initAppPluginRoutes(r *macaron.Macaron) {
}
}
handlers = append(handlers, AppPluginRoute(route, plugin.Id, hs))
- r.Handle(route.Method, url, handlers)
+ for _, method := range strings.Split(route.Method, ",") {
+ r.Handle(strings.TrimSpace(method), url, handlers)
+ }
log.Debugf("Plugins: Adding proxy route %s", url)
}
}
diff --git a/pkg/api/datasources_test.go b/pkg/api/datasources_test.go
index cde05b6fd2e..5103d3b5fa2 100644
--- a/pkg/api/datasources_test.go
+++ b/pkg/api/datasources_test.go
@@ -234,7 +234,7 @@ func TestAPI_Datasources_AccessControl(t *testing.T) {
desc: "DatasourcesGet should return 200 for user with correct permissions",
url: "/api/datasources/",
method: http.MethodGet,
- permissions: []*accesscontrol.Permission{{Action: ActionDatasourcesRead}},
+ permissions: []*accesscontrol.Permission{{Action: ActionDatasourcesRead, Scope: ScopeDatasourcesAll}},
},
},
{
diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go
index 3c8e8b68509..1ef74759721 100644
--- a/pkg/api/frontendsettings.go
+++ b/pkg/api/frontendsettings.go
@@ -5,6 +5,7 @@ import (
"strconv"
"github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/tsdb/grafanads"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/util"
@@ -112,11 +113,16 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins *plu
// the datasource table)
for _, ds := range hs.PluginManager.DataSources() {
if ds.BuiltIn {
- dataSources[ds.Name] = map[string]interface{}{
+ info := map[string]interface{}{
"type": ds.Type,
"name": ds.Name,
"meta": hs.PluginManager.GetDataSource(ds.Id),
}
+ if ds.Name == grafanads.DatasourceName {
+ info["id"] = grafanads.DatasourceID
+ info["uid"] = grafanads.DatasourceUID
+ }
+ dataSources[ds.Name] = info
}
}
diff --git a/pkg/api/index.go b/pkg/api/index.go
index cf65f7bfd31..ece4275885c 100644
--- a/pkg/api/index.go
+++ b/pkg/api/index.go
@@ -18,6 +18,16 @@ const (
darkName = "dark"
)
+// dataSourcesConfigurationAccessEvaluator is used to protect the "Configure > Data sources" tab access
+var dataSourcesConfigurationAccessEvaluator = ac.EvalAll(
+ ac.EvalPermission(ActionDatasourcesRead, ScopeDatasourcesAll),
+ ac.EvalAny(
+ ac.EvalPermission(ActionDatasourcesCreate),
+ ac.EvalPermission(ActionDatasourcesDelete),
+ ac.EvalPermission(ActionDatasourcesWrite),
+ ),
+)
+
func (hs *HTTPServer) getProfileNode(c *models.ReqContext) *dtos.NavLink {
// Only set login if it's different from the name
var login string
@@ -253,7 +263,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto
configNodes := []*dtos.NavLink{}
- if hasAccess(ac.ReqOrgAdmin, ac.EvalPermission(ActionDatasourcesRead)) {
+ if hasAccess(ac.ReqOrgAdmin, dataSourcesConfigurationAccessEvaluator) {
configNodes = append(configNodes, &dtos.NavLink{
Text: "Data sources",
Icon: "database",
@@ -308,6 +318,30 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dto
})
}
+ if true {
+ liveNavLinks := []*dtos.NavLink{}
+
+ liveNavLinks = append(liveNavLinks, &dtos.NavLink{
+ Text: "Status", Id: "live-status", Url: hs.Cfg.AppSubURL + "/live", Icon: "exchange-alt",
+ })
+ liveNavLinks = append(liveNavLinks, &dtos.NavLink{
+ Text: "Pipeline", Id: "live-pipeline", Url: hs.Cfg.AppSubURL + "/live/pipeline", Icon: "arrow-to-right",
+ })
+ liveNavLinks = append(liveNavLinks, &dtos.NavLink{
+ Text: "Cloud", Id: "live-cloud", Url: hs.Cfg.AppSubURL + "/live/cloud", Icon: "cloud-upload",
+ })
+
+ navTree = append(navTree, &dtos.NavLink{
+ Id: "live",
+ Text: "Live",
+ SubTitle: "Event Streaming",
+ Icon: "exchange-alt",
+ Url: hs.Cfg.AppSubURL + "/live",
+ Children: liveNavLinks,
+ HideFromMenu: true,
+ })
+ }
+
if len(configNodes) > 0 {
navTree = append(navTree, &dtos.NavLink{
Id: dtos.NavIDCfg,
diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go
index 6a60543fa46..d99940ad0d1 100644
--- a/pkg/api/metrics.go
+++ b/pkg/api/metrics.go
@@ -1,19 +1,17 @@
package api
import (
- "context"
"errors"
"net/http"
+ "github.com/grafana/grafana/pkg/tsdb/grafanads"
+
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/api/response"
- "github.com/grafana/grafana/pkg/bus"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
- "github.com/grafana/grafana/pkg/util"
)
// QueryMetricsV2 returns query metrics.
@@ -32,31 +30,42 @@ func (hs *HTTPServer) QueryMetricsV2(c *models.ReqContext, reqDTO dtos.MetricReq
}
// Loop to see if we have an expression.
+ prevType := ""
+ var ds *models.DataSource
for _, query := range reqDTO.Queries {
- if query.Get("datasource").MustString("") == expr.DatasourceName {
+ dsType := query.Get("datasource").MustString("")
+ if dsType == expr.DatasourceName {
return hs.handleExpressions(c, reqDTO)
}
- }
-
- var ds *models.DataSource
- for i, query := range reqDTO.Queries {
- hs.log.Debug("Processing metrics query", "query", query)
-
- datasourceID, err := query.Get("datasourceId").Int64()
- if err != nil {
+ if prevType != "" && prevType != dsType {
+ // For mixed datasource case, each data source is sent in a single request.
+ // So only the datasource from the first query is needed. As all requests
+ // should be the same data source.
hs.log.Debug("Can't process query since it's missing data source ID")
- return response.Error(http.StatusBadRequest, "Query missing data source ID", nil)
+ return response.Error(http.StatusBadRequest, "All queries must use the same datasource", nil)
}
- // For mixed datasource case, each data source is sent in a single request.
- // So only the datasource from the first query is needed. As all requests
- // should be the same data source.
- if i == 0 {
- ds, err = hs.DataSourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache)
+ if ds == nil {
+ // require ID for everything
+ dsID, err := query.Get("datasourceId").Int64()
if err != nil {
- return hs.handleGetDataSourceError(err, datasourceID)
+ hs.log.Debug("Can't process query since it's missing data source ID")
+ return response.Error(http.StatusBadRequest, "Query missing data source ID", nil)
+ }
+ if dsID == grafanads.DatasourceID {
+ ds = grafanads.DataSourceModel(c.OrgId)
+ } else {
+ ds, err = hs.DataSourceCache.GetDatasource(dsID, c.SignedInUser, c.SkipCache)
+ if err != nil {
+ return hs.handleGetDataSourceError(err, dsID)
+ }
}
}
+ prevType = dsType
+ }
+
+ for _, query := range reqDTO.Queries {
+ hs.log.Debug("Processing metrics query", "query", query)
request.Queries = append(request.Queries, plugins.DataSubQuery{
RefID: query.Get("refId").MustString("A"),
@@ -213,46 +222,3 @@ func (hs *HTTPServer) QueryMetrics(c *models.ReqContext, reqDto dtos.MetricReque
return response.JSON(statusCode, &resp)
}
-
-// GET /api/tsdb/testdata/gensql
-func GenerateSQLTestData(c *models.ReqContext) response.Response {
- if err := bus.Dispatch(&models.InsertSQLTestDataCommand{}); err != nil {
- return response.Error(500, "Failed to insert test data", err)
- }
-
- return response.JSON(200, &util.DynMap{"message": "OK"})
-}
-
-// GET /api/tsdb/testdata/random-walk
-func (hs *HTTPServer) GetTestDataRandomWalk(c *models.ReqContext) response.Response {
- from := c.Query("from")
- to := c.Query("to")
- intervalMS := c.QueryInt64("intervalMs")
-
- timeRange := plugins.NewDataTimeRange(from, to)
- request := plugins.DataQuery{TimeRange: &timeRange}
-
- dsInfo := &models.DataSource{
- Type: "testdata",
- JsonData: simplejson.New(),
- }
- request.Queries = append(request.Queries, plugins.DataSubQuery{
- RefID: "A",
- IntervalMS: intervalMS,
- Model: simplejson.NewFromAny(&util.DynMap{
- "scenario": "random_walk",
- }),
- DataSource: dsInfo,
- })
-
- resp, err := hs.DataService.HandleRequest(context.Background(), dsInfo, request)
- if err != nil {
- return response.Error(500, "Metric request error", err)
- }
-
- qdr, err := resp.ToBackendDataResponse()
- if err != nil {
- return response.Error(http.StatusInternalServerError, "error converting results", err)
- }
- return toMacronResponse(qdr)
-}
diff --git a/pkg/api/plugins.go b/pkg/api/plugins.go
index 7ece17c06cf..65f895f753d 100644
--- a/pkg/api/plugins.go
+++ b/pkg/api/plugins.go
@@ -3,12 +3,10 @@ package api
import (
"encoding/json"
"errors"
- "fmt"
"net/http"
"os"
"path/filepath"
"sort"
- "strings"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/api/dtos"
@@ -21,14 +19,6 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
-var permittedFileExts = []string{
- ".html", ".xhtml", ".css", ".js", ".json", ".jsonld", ".map", ".mjs",
- ".jpeg", ".jpg", ".png", ".gif", ".svg", ".webp", ".ico",
- ".woff", ".woff2", ".eot", ".ttf", ".otf",
- ".wav", ".mp3",
- ".md", ".pdf", ".txt",
-}
-
func (hs *HTTPServer) GetPluginList(c *models.ReqContext) response.Response {
typeFilter := c.Query("type")
enabledFilter := c.Query("enabled")
@@ -262,10 +252,10 @@ func (hs *HTTPServer) CollectPluginMetrics(c *models.ReqContext) response.Respon
return response.CreateNormalResponse(headers, resp.PrometheusMetrics, http.StatusOK)
}
-// GetPluginAssets returns public plugin assets (images, JS, etc.)
+// getPluginAssets returns public plugin assets (images, JS, etc.)
//
// /public/plugins/:pluginId/*
-func (hs *HTTPServer) GetPluginAssets(c *models.ReqContext) {
+func (hs *HTTPServer) getPluginAssets(c *models.ReqContext) {
pluginID := c.Params("pluginId")
plugin := hs.PluginManager.GetPlugin(pluginID)
if plugin == nil {
@@ -276,6 +266,11 @@ func (hs *HTTPServer) GetPluginAssets(c *models.ReqContext) {
requestedFile := filepath.Clean(c.Params("*"))
pluginFilePath := filepath.Join(plugin.PluginDir, requestedFile)
+ if !plugin.IncludedInSignature(requestedFile) {
+ hs.log.Warn("Access to requested plugin file will be forbidden in upcoming Grafana versions as the file "+
+ "is not included in the plugin signature", "file", requestedFile)
+ }
+
// It's safe to ignore gosec warning G304 since we already clean the requested file path and subsequently
// use this with a prefix of the plugin's directory, which is set during plugin loading
// nolint:gosec
@@ -300,12 +295,6 @@ func (hs *HTTPServer) GetPluginAssets(c *models.ReqContext) {
return
}
- if accessForbidden(fi.Name()) {
- c.JsonApiErr(403, "Plugin file access forbidden",
- fmt.Errorf("access is forbidden to plugin file %s", pluginFilePath))
- return
- }
-
if hs.Cfg.Env == setting.Dev {
c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache")
} else {
@@ -448,14 +437,3 @@ func translatePluginRequestErrorToAPIError(err error) response.Response {
return response.Error(500, "Plugin request failed", err)
}
-
-func accessForbidden(pluginFilename string) bool {
- ext := filepath.Ext(pluginFilename)
-
- for _, permittedExt := range permittedFileExts {
- if strings.EqualFold(permittedExt, ext) {
- return false
- }
- }
- return true
-}
diff --git a/pkg/api/plugins_test.go b/pkg/api/plugins_test.go
index 47ea607cda6..2814bea3ad1 100644
--- a/pkg/api/plugins_test.go
+++ b/pkg/api/plugins_test.go
@@ -1,89 +1,201 @@
package api
import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
"testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/grafana/grafana/pkg/bus"
+ "github.com/grafana/grafana/pkg/infra/log"
+ "github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/plugins"
+ "github.com/grafana/grafana/pkg/plugins/manager"
+ "github.com/grafana/grafana/pkg/setting"
)
-func Test_accessForbidden(t *testing.T) {
- type testCase struct {
- filename string
- }
- tests := []struct {
- name string
- t testCase
- accessForbidden bool
- }{
- {
- name: ".exe files are forbidden",
- t: testCase{
- filename: "test.exe",
- },
- accessForbidden: true,
- },
- {
- name: ".sh files are forbidden",
- t: testCase{
- filename: "test.sh",
- },
- accessForbidden: true,
- },
- {
- name: "js is not forbidden",
- t: testCase{
+func Test_GetPluginAssets(t *testing.T) {
+ pluginID := "test-plugin"
+ pluginDir := "."
+ tmpFile, err := ioutil.TempFile(pluginDir, "")
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ err := os.RemoveAll(tmpFile.Name())
+ assert.NoError(t, err)
+ })
+ expectedBody := "Plugin test"
+ _, err = tmpFile.WriteString(expectedBody)
+ assert.NoError(t, err)
- filename: "module.js",
- },
- accessForbidden: false,
- },
- {
- name: "logos are not forbidden",
- t: testCase{
+ requestedFile := filepath.Clean(tmpFile.Name())
- filename: "logo.svg",
+ t.Run("Given a request for an existing plugin file that is listed as a signature covered file", func(t *testing.T) {
+ p := &plugins.PluginBase{
+ Id: pluginID,
+ PluginDir: pluginDir,
+ SignedFiles: map[string]struct{}{
+ requestedFile: {},
},
- accessForbidden: false,
- },
- {
- name: "JPGs are not forbidden",
- t: testCase{
- filename: "img/test.jpg",
+ }
+ service := &pluginManager{
+ plugins: map[string]*plugins.PluginBase{
+ pluginID: p,
},
- accessForbidden: false,
- },
- {
- name: "JPEGs are not forbidden",
- t: testCase{
- filename: "img/test.jpeg",
+ }
+ l := &logger{}
+
+ url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
+ pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", service, l,
+ func(sc *scenarioContext) {
+ callGetPluginAsset(sc)
+
+ require.Equal(t, 200, sc.resp.Code)
+ assert.Equal(t, expectedBody, sc.resp.Body.String())
+ assert.Empty(t, l.warnings)
+ })
+ })
+
+ t.Run("Given a request for an existing plugin file that is not listed as a signature covered file", func(t *testing.T) {
+ p := &plugins.PluginBase{
+ Id: pluginID,
+ PluginDir: pluginDir,
+ }
+ service := &pluginManager{
+ plugins: map[string]*plugins.PluginBase{
+ pluginID: p,
},
- accessForbidden: false,
- },
- {
- name: "ext case is ignored",
- t: testCase{
- filename: "scripts/runThis.SH",
+ }
+ l := &logger{}
+
+ url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
+ pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", service, l,
+ func(sc *scenarioContext) {
+ callGetPluginAsset(sc)
+
+ require.Equal(t, 200, sc.resp.Code)
+ assert.Equal(t, expectedBody, sc.resp.Body.String())
+ assert.Empty(t, l.warnings)
+ })
+ })
+
+ t.Run("Given a request for an non-existing plugin file", func(t *testing.T) {
+ p := &plugins.PluginBase{
+ Id: pluginID,
+ PluginDir: pluginDir,
+ }
+ service := &pluginManager{
+ plugins: map[string]*plugins.PluginBase{
+ pluginID: p,
},
- accessForbidden: true,
- },
- {
- name: "no file ext is forbidden",
- t: testCase{
- filename: "scripts/runThis",
+ }
+ l := &logger{}
+
+ requestedFile := "nonExistent"
+ url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
+ pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", service, l,
+ func(sc *scenarioContext) {
+ callGetPluginAsset(sc)
+
+ var respJson map[string]interface{}
+ err := json.NewDecoder(sc.resp.Body).Decode(&respJson)
+ require.NoError(t, err)
+ require.Equal(t, 404, sc.resp.Code)
+ assert.Equal(t, "Plugin file not found", respJson["message"])
+ assert.Empty(t, l.warnings)
+ })
+ })
+
+ t.Run("Given a request for an non-existing plugin", func(t *testing.T) {
+ service := &pluginManager{
+ plugins: map[string]*plugins.PluginBase{},
+ }
+ l := &logger{}
+
+ requestedFile := "nonExistent"
+ url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
+ pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", service, l,
+ func(sc *scenarioContext) {
+ callGetPluginAsset(sc)
+
+ var respJson map[string]interface{}
+ err := json.NewDecoder(sc.resp.Body).Decode(&respJson)
+ require.NoError(t, err)
+ assert.Equal(t, 404, sc.resp.Code)
+ assert.Equal(t, "Plugin not found", respJson["message"])
+ assert.Empty(t, l.warnings)
+ })
+ })
+
+ t.Run("Given a request for a core plugin's file", func(t *testing.T) {
+ service := &pluginManager{
+ plugins: map[string]*plugins.PluginBase{
+ pluginID: {
+ IsCorePlugin: true,
+ },
},
- accessForbidden: true,
- },
- {
- name: "empty file ext is forbidden",
- t: testCase{
- filename: "scripts/runThis.",
- },
- accessForbidden: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if got := accessForbidden(tt.t.filename); got != tt.accessForbidden {
- t.Errorf("accessForbidden() = %v, accessForbidden %v", got, tt.accessForbidden)
- }
- })
- }
+ }
+ l := &logger{}
+
+ url := fmt.Sprintf("/public/plugins/%s/%s", pluginID, requestedFile)
+ pluginAssetScenario(t, "When calling GET on", url, "/public/plugins/:pluginId/*", service, l,
+ func(sc *scenarioContext) {
+ callGetPluginAsset(sc)
+
+ require.Equal(t, 200, sc.resp.Code)
+ assert.Equal(t, expectedBody, sc.resp.Body.String())
+ assert.Empty(t, l.warnings)
+ })
+ })
+}
+
+func callGetPluginAsset(sc *scenarioContext) {
+ sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec()
+}
+
+func pluginAssetScenario(t *testing.T, desc string, url string, urlPattern string, pluginManager plugins.Manager,
+ logger log.Logger, fn scenarioFunc) {
+ t.Run(fmt.Sprintf("%s %s", desc, url), func(t *testing.T) {
+ defer bus.ClearBusHandlers()
+
+ hs := HTTPServer{
+ Cfg: setting.NewCfg(),
+ PluginManager: pluginManager,
+ log: logger,
+ }
+
+ sc := setupScenarioContext(t, url)
+ sc.defaultHandler = func(c *models.ReqContext) {
+ sc.context = c
+ hs.getPluginAssets(c)
+ }
+
+ sc.m.Get(urlPattern, sc.defaultHandler)
+
+ fn(sc)
+ })
+}
+
+type pluginManager struct {
+ manager.PluginManager
+
+ plugins map[string]*plugins.PluginBase
+}
+
+func (pm *pluginManager) GetPlugin(id string) *plugins.PluginBase {
+ return pm.plugins[id]
+}
+
+type logger struct {
+ log.Logger
+
+ warnings []string
+}
+
+func (l *logger) Warn(msg string, ctx ...interface{}) {
+ l.warnings = append(l.warnings, msg)
}
diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go
new file mode 100644
index 00000000000..5817cee0657
--- /dev/null
+++ b/pkg/build/cmd.go
@@ -0,0 +1,316 @@
+package build
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "go/build"
+ "io/ioutil"
+ "log"
+ "os"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ GoOSWindows = "windows"
+ GoOSLinux = "linux"
+
+ ServerBinary = "grafana-server"
+ CLIBinary = "grafana-cli"
+)
+
+var binaries = []string{ServerBinary, CLIBinary}
+
+func logError(message string, err error) int {
+ log.Println(message, err)
+
+ return 1
+}
+
+// RunCmd runs the build command and returns the exit code
+func RunCmd() int {
+ opts := BuildOptsFromFlags()
+
+ wd, err := os.Getwd()
+ if err != nil {
+ return logError("Error getting working directory", err)
+ }
+
+ packageJSON, err := OpenPackageJSON(wd)
+ if err != nil {
+ return logError("Error opening package json", err)
+ }
+
+ opts.version = packageJSON.Version
+
+ version, iteration := LinuxPackageVersion(packageJSON.Version, opts.buildID)
+
+ if opts.printGenVersion {
+ fmt.Print(genPackageVersion(version, iteration))
+ return 0
+ }
+
+ log.Printf("Version: %s, Linux Version: %s, Package Iteration: %s\n", version, version, iteration)
+
+ if flag.NArg() == 0 {
+ log.Println("Usage: go run build.go build")
+ return 1
+ }
+
+ for _, cmd := range flag.Args() {
+ switch cmd {
+ case "setup":
+ setup(opts.goos)
+
+ case "build-srv", "build-server":
+ if !opts.isDev {
+ clean(opts)
+ }
+
+ if err := doBuild("grafana-server", "./pkg/cmd/grafana-server", opts); err != nil {
+ log.Println(err)
+ return 1
+ }
+
+ case "build-cli":
+ clean(opts)
+ if err := doBuild("grafana-cli", "./pkg/cmd/grafana-cli", opts); err != nil {
+ log.Println(err)
+ return 1
+ }
+
+ case "build":
+ //clean()
+ for _, binary := range binaries {
+ log.Println("building binaries", cmd)
+ // Can't use filepath.Join here because filepath.Join calls filepath.Clean, which removes the `./` from this path, which upsets `go build`
+ if err := doBuild(binary, fmt.Sprintf("./pkg/cmd/%s", binary), opts); err != nil {
+ log.Println(err)
+ return 1
+ }
+ }
+
+ case "build-frontend":
+ yarn("build")
+
+ case "sha-dist":
+ if err := shaDir("dist"); err != nil {
+ return logError("error packaging dist directory", err)
+ }
+
+ case "latest":
+ makeLatestDistCopies()
+
+ case "clean":
+ clean(opts)
+
+ default:
+ log.Println("Unknown command", cmd)
+ return 1
+ }
+ }
+
+ return 0
+}
+
+func makeLatestDistCopies() {
+ files, err := ioutil.ReadDir("dist")
+ if err != nil {
+ log.Fatalf("failed to create latest copies. Cannot read from /dist")
+ }
+
+ latestMapping := map[string]string{
+ "_amd64.deb": "dist/grafana_latest_amd64.deb",
+ ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm",
+ ".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
+ ".linux-amd64-musl.tar.gz": "dist/grafana-latest.linux-x64-musl.tar.gz",
+ ".linux-armv7.tar.gz": "dist/grafana-latest.linux-armv7.tar.gz",
+ ".linux-armv7-musl.tar.gz": "dist/grafana-latest.linux-armv7-musl.tar.gz",
+ ".linux-armv6.tar.gz": "dist/grafana-latest.linux-armv6.tar.gz",
+ ".linux-arm64.tar.gz": "dist/grafana-latest.linux-arm64.tar.gz",
+ ".linux-arm64-musl.tar.gz": "dist/grafana-latest.linux-arm64-musl.tar.gz",
+ }
+
+ for _, file := range files {
+ for extension, fullName := range latestMapping {
+ if strings.HasSuffix(file.Name(), extension) {
+ if _, err := runError("cp", path.Join("dist", file.Name()), fullName); err != nil {
+ log.Println("error running cp command:", err)
+ }
+ }
+ }
+ }
+}
+
+func yarn(params ...string) {
+ runPrint(`yarn run`, params...)
+}
+
+func genPackageVersion(version string, iteration string) string {
+ if iteration != "" {
+ return fmt.Sprintf("%v-%v", version, iteration)
+ } else {
+ return version
+ }
+}
+
+func setup(goos string) {
+ args := []string{"install", "-v"}
+ if goos == GoOSWindows {
+ args = append(args, "-buildmode=exe")
+ }
+ args = append(args, "./pkg/cmd/grafana-server")
+ runPrint("go", args...)
+}
+
+func doBuild(binaryName, pkg string, opts BuildOpts) error {
+ log.Println("building", binaryName, pkg)
+ libcPart := ""
+ if opts.libc != "" {
+ libcPart = fmt.Sprintf("-%s", opts.libc)
+ }
+ binary := fmt.Sprintf("./bin/%s", binaryName)
+
+ //don't include os/arch/libc in output path in dev environment
+ if !opts.isDev {
+ binary = fmt.Sprintf("./bin/%s-%s%s/%s", opts.goos, opts.goarch, libcPart, binaryName)
+ }
+
+ if opts.goos == GoOSWindows {
+ binary += ".exe"
+ }
+
+ if !opts.isDev {
+ rmr(binary, binary+".md5")
+ }
+
+ lf, err := ldflags(opts)
+ if err != nil {
+ return err
+ }
+
+ args := []string{"build", "-ldflags", lf}
+
+ if opts.goos == GoOSWindows {
+ // Work around a linking error on Windows: "export ordinal too large"
+ args = append(args, "-buildmode=exe")
+ }
+
+ if len(opts.buildTags) > 0 {
+ args = append(args, "-tags", strings.Join(opts.buildTags, ","))
+ }
+
+ if opts.race {
+ args = append(args, "-race")
+ }
+
+ args = append(args, "-o", binary)
+ args = append(args, pkg)
+
+ runPrint("go", args...)
+
+ if opts.isDev {
+ return nil
+ }
+
+ if err := setBuildEnv(opts); err != nil {
+ return err
+ }
+ runPrint("go", "version")
+ libcPart = ""
+ if opts.libc != "" {
+ libcPart = fmt.Sprintf("/%s", opts.libc)
+ }
+ fmt.Printf("Targeting %s/%s%s\n", opts.goos, opts.goarch, libcPart)
+
+ // Create an md5 checksum of the binary, to be included in the archive for
+ // automatic upgrades.
+ return md5File(binary)
+}
+
+func ldflags(opts BuildOpts) (string, error) {
+ buildStamp, err := buildStamp()
+ if err != nil {
+ return "", err
+ }
+
+ var b bytes.Buffer
+ b.WriteString("-w")
+ b.WriteString(fmt.Sprintf(" -X main.version=%s", opts.version))
+ b.WriteString(fmt.Sprintf(" -X main.commit=%s", getGitSha()))
+ b.WriteString(fmt.Sprintf(" -X main.buildstamp=%d", buildStamp))
+ b.WriteString(fmt.Sprintf(" -X main.buildBranch=%s", getGitBranch()))
+ if v := os.Getenv("LDFLAGS"); v != "" {
+ b.WriteString(fmt.Sprintf(" -extldflags \"%s\"", v))
+ }
+
+ return b.String(), nil
+}
+
+func setBuildEnv(opts BuildOpts) error {
+ if err := os.Setenv("GOOS", opts.goos); err != nil {
+ return err
+ }
+
+ if opts.goos == GoOSWindows {
+ // require windows >=7
+ if err := os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601"); err != nil {
+ return err
+ }
+ }
+
+ if opts.goarch != "amd64" || opts.goos != GoOSLinux {
+ // needed for all other archs
+ opts.cgo = true
+ }
+
+ if strings.HasPrefix(opts.goarch, "armv") {
+ if err := os.Setenv("GOARCH", "arm"); err != nil {
+ return err
+ }
+
+ if err := os.Setenv("GOARM", opts.goarch[4:]); err != nil {
+ return err
+ }
+ } else {
+ if err := os.Setenv("GOARCH", opts.goarch); err != nil {
+ return err
+ }
+ }
+
+ if opts.cgo {
+ if err := os.Setenv("CGO_ENABLED", "1"); err != nil {
+ return err
+ }
+ }
+
+ if opts.gocc == "" {
+ return nil
+ }
+
+ return os.Setenv("CC", opts.gocc)
+}
+
+func buildStamp() (int64, error) {
+ // use SOURCE_DATE_EPOCH if set.
+ if v, ok := os.LookupEnv("SOURCE_DATE_EPOCH"); ok {
+ return strconv.ParseInt(v, 10, 64)
+ }
+
+ bs, err := runError("git", "show", "-s", "--format=%ct")
+ if err != nil {
+ return time.Now().Unix(), nil
+ }
+
+ return strconv.ParseInt(string(bs), 10, 64)
+}
+
+func clean(opts BuildOpts) {
+ rmr("dist")
+ rmr("tmp")
+ rmr(filepath.Join(build.Default.GOPATH, fmt.Sprintf("pkg/%s_%s/github.com/grafana", opts.goos, opts.goarch)))
+}
diff --git a/pkg/build/docs.go b/pkg/build/docs.go
new file mode 100644
index 00000000000..ca9549e8799
--- /dev/null
+++ b/pkg/build/docs.go
@@ -0,0 +1,2 @@
+// Package build contains the command / functions for the Grafana build process used when running the "build" target in the makefile
+package build
diff --git a/pkg/build/exec.go b/pkg/build/exec.go
new file mode 100644
index 00000000000..ad13eb5ba99
--- /dev/null
+++ b/pkg/build/exec.go
@@ -0,0 +1,34 @@
+package build
+
+import (
+ "bytes"
+ "log"
+ "os"
+ "os/exec"
+ "strings"
+)
+
+func runError(cmd string, args ...string) ([]byte, error) {
+ // Can ignore gosec G204 because this function is not used in Grafana, only in the build process.
+ //nolint:gosec
+ ecmd := exec.Command(cmd, args...)
+ bs, err := ecmd.CombinedOutput()
+ if err != nil {
+ return nil, err
+ }
+
+ return bytes.TrimSpace(bs), nil
+}
+
+func runPrint(cmd string, args ...string) {
+ log.Println(cmd, strings.Join(args, " "))
+ // Can ignore gosec G204 because this function is not used in Grafana, only in the build process.
+ //nolint:gosec
+ ecmd := exec.Command(cmd, args...)
+ ecmd.Stdout = os.Stdout
+ ecmd.Stderr = os.Stderr
+ err := ecmd.Run()
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/pkg/build/fs.go b/pkg/build/fs.go
new file mode 100644
index 00000000000..fe1eaba2577
--- /dev/null
+++ b/pkg/build/fs.go
@@ -0,0 +1,101 @@
+package build
+
+import (
+ "crypto/md5"
+ "crypto/sha256"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+func logAndClose(c io.Closer) {
+ if err := c.Close(); err != nil {
+ log.Println("error closing:", err)
+ }
+}
+
+func shaDir(dir string) error {
+ return filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
+ if path == dir {
+ return nil
+ }
+
+ if strings.Contains(path, ".sha256") {
+ return nil
+ }
+ if err := shaFile(path); err != nil {
+ log.Printf("Failed to create sha file. error: %v\n", err)
+ }
+ return nil
+ })
+}
+
+func shaFile(file string) error {
+ // Can ignore gosec G304 because this function is not used in Grafana, only in the build process.
+ //nolint:gosec
+ r, err := os.Open(file)
+ if err != nil {
+ return err
+ }
+
+ defer logAndClose(r)
+
+ h := sha256.New()
+ _, err = io.Copy(h, r)
+ if err != nil {
+ return err
+ }
+
+ out, err := os.Create(file + ".sha256")
+ if err != nil {
+ return err
+ }
+
+ _, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
+ if err != nil {
+ return err
+ }
+
+ return out.Close()
+}
+
+func md5File(file string) error {
+ // Can ignore gosec G304 because this function is not used in Grafana, only in the build process.
+ //nolint:gosec
+ fd, err := os.Open(file)
+ if err != nil {
+ return err
+ }
+ defer logAndClose(fd)
+
+ h := md5.New()
+ _, err = io.Copy(h, fd)
+ if err != nil {
+ return err
+ }
+
+ out, err := os.Create(file + ".md5")
+ if err != nil {
+ return err
+ }
+
+ _, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
+ if err != nil {
+ return err
+ }
+
+ return out.Close()
+}
+
+// basically `rm -r`s the list of files provided
+func rmr(paths ...string) {
+ for _, path := range paths {
+ log.Println("rm -r", path)
+ if err := os.RemoveAll(path); err != nil {
+ log.Println("error deleting folder", path, "error:", err)
+ }
+ }
+}
diff --git a/pkg/build/git.go b/pkg/build/git.go
new file mode 100644
index 00000000000..6c86d74e004
--- /dev/null
+++ b/pkg/build/git.go
@@ -0,0 +1,17 @@
+package build
+
+func getGitBranch() string {
+ v, err := runError("git", "rev-parse", "--abbrev-ref", "HEAD")
+ if err != nil {
+ return "main"
+ }
+ return string(v)
+}
+
+func getGitSha() string {
+ v, err := runError("git", "rev-parse", "--short", "HEAD")
+ if err != nil {
+ return "unknown-dev"
+ }
+ return string(v)
+}
diff --git a/pkg/build/opts.go b/pkg/build/opts.go
new file mode 100644
index 00000000000..8e5486d22d1
--- /dev/null
+++ b/pkg/build/opts.go
@@ -0,0 +1,66 @@
+package build
+
+import (
+ "flag"
+ "runtime"
+ "strings"
+)
+
+// BuildOpts are options provided to the build step
+type BuildOpts struct {
+ goarch string
+ goos string
+ gocc string
+ cgo bool
+ libc string
+
+ pkgArch string
+ version string
+ buildTags []string
+ // deb & rpm does not support semver so have to handle their version a little differently
+ race bool
+ includeBuildID bool
+ buildID string
+ isDev bool
+ enterprise bool
+ skipRpmGen bool
+ skipDebGen bool
+ printGenVersion bool
+}
+
+// BuildOptsFromFlags reads the cmd args to assemble a BuildOpts object. This function calls flag.Parse()
+func BuildOptsFromFlags() BuildOpts {
+ opts := BuildOpts{}
+
+ var buildIDRaw string
+ var buildTagsRaw string
+
+ flag.StringVar(&opts.goarch, "goarch", runtime.GOARCH, "GOARCH")
+ flag.StringVar(&opts.goos, "goos", runtime.GOOS, "GOOS")
+ flag.StringVar(&opts.gocc, "cc", "", "CC")
+ flag.StringVar(&opts.libc, "libc", "", "LIBC")
+ flag.StringVar(&buildTagsRaw, "build-tags", "", "Sets custom build tags")
+ flag.BoolVar(&opts.cgo, "cgo-enabled", false, "Enable cgo")
+ flag.StringVar(&opts.pkgArch, "pkg-arch", "", "PKG ARCH")
+ flag.BoolVar(&opts.race, "race", false, "Use race detector")
+ flag.BoolVar(&opts.includeBuildID, "includeBuildID", true, "IncludeBuildID in package name")
+ flag.BoolVar(&opts.enterprise, "enterprise", false, "Build enterprise version of Grafana")
+ flag.StringVar(&buildIDRaw, "buildID", "0", "Build ID from CI system")
+ flag.BoolVar(&opts.isDev, "dev", false, "optimal for development, skips certain steps")
+ flag.BoolVar(&opts.skipRpmGen, "skipRpm", false, "skip rpm package generation (default: false)")
+ flag.BoolVar(&opts.skipDebGen, "skipDeb", false, "skip deb package generation (default: false)")
+ flag.BoolVar(&opts.printGenVersion, "gen-version", false, "generate Grafana version and output (default: false)")
+ flag.Parse()
+
+ opts.buildID = shortenBuildID(buildIDRaw)
+
+ if len(buildTagsRaw) > 0 {
+ opts.buildTags = strings.Split(buildTagsRaw, ",")
+ }
+
+ if opts.pkgArch == "" {
+ opts.pkgArch = opts.goarch
+ }
+
+ return opts
+}
diff --git a/pkg/build/version.go b/pkg/build/version.go
new file mode 100644
index 00000000000..9ecb6b5b972
--- /dev/null
+++ b/pkg/build/version.go
@@ -0,0 +1,66 @@
+package build
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "strings"
+ "time"
+)
+
+type PackageJSON struct {
+ Version string `json:"version"`
+}
+
+// Opens the package.json file in the provided directory and returns a struct that represents its contents
+func OpenPackageJSON(dir string) (PackageJSON, error) {
+ reader, err := os.Open("package.json")
+ if err != nil {
+ return PackageJSON{}, err
+ }
+
+ defer logAndClose(reader)
+
+ jsonObj := PackageJSON{}
+ if err := json.NewDecoder(reader).Decode(&jsonObj); err != nil {
+ return PackageJSON{}, err
+ }
+
+ return jsonObj, nil
+}
+
+// LinuxPackageVersion extracts the linux package version and iteration out of the version string. The version string is likely extracted from the package JSON.
+func LinuxPackageVersion(v string, buildID string) (string, string) {
+ var (
+ version = v
+ iteration = ""
+ )
+
+ // handle pre version stuff (deb / rpm does not support semver)
+ parts := strings.Split(v, "-")
+
+ if len(parts) > 1 {
+ version = parts[0]
+ iteration = parts[1]
+ }
+
+ if buildID == "" {
+ return version, iteration
+ }
+
+ // add timestamp to iteration
+ if buildID != "0" {
+ iteration = strings.Join([]string{buildID, iteration}, "")
+ return version, iteration
+ }
+
+ return version, fmt.Sprintf("%d%s", time.Now().Unix(), iteration)
+}
+
+func shortenBuildID(buildID string) string {
+ buildID = strings.Replace(buildID, "-", "", -1)
+ if len(buildID) < 9 {
+ return buildID
+ }
+ return buildID[0:8]
+}
diff --git a/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go b/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go
index 97833207533..5f8edc910a1 100644
--- a/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go
+++ b/pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go
@@ -28,6 +28,7 @@ func TestValidateScuemataBasics(t *testing.T) {
})
t.Run("Testing scuemata validity with invalid cue schemas - family missing", func(t *testing.T) {
+ t.Skip() // TODO debug, re-enable and move
genCue, err := os.ReadFile("testdata/missing_family.cue")
require.NoError(t, err)
@@ -46,6 +47,7 @@ func TestValidateScuemataBasics(t *testing.T) {
})
t.Run("Testing scuemata validity with invalid cue schemas - panel missing ", func(t *testing.T) {
+ t.Skip() // TODO debug, re-enable and move
genCue, err := os.ReadFile("testdata/missing_panel.cue")
require.NoError(t, err)
diff --git a/pkg/cmd/grafana-cli/commands/testdata/missing_family.cue b/pkg/cmd/grafana-cli/commands/testdata/missing_family.cue
index 7d008ab7507..f4913630f34 100644
--- a/pkg/cmd/grafana-cli/commands/testdata/missing_family.cue
+++ b/pkg/cmd/grafana-cli/commands/testdata/missing_family.cue
@@ -1,4 +1,4 @@
-package grafanaschema
+package dashboard
import "github.com/grafana/grafana/cue/scuemata"
@@ -76,8 +76,3 @@ Dummy: scuemata.#Family & {
]
]
}
-
-#Latest: {
- #Dashboard: Dummy.latest
- #Panel: Dummy.latest._Panel
-}
diff --git a/pkg/cmd/grafana-cli/commands/testdata/missing_panel.cue b/pkg/cmd/grafana-cli/commands/testdata/missing_panel.cue
index 3d3cbe42ba9..07aa256bff0 100644
--- a/pkg/cmd/grafana-cli/commands/testdata/missing_panel.cue
+++ b/pkg/cmd/grafana-cli/commands/testdata/missing_panel.cue
@@ -1,4 +1,4 @@
-package grafanaschema
+package dashboard
import "github.com/grafana/grafana/cue/scuemata"
@@ -76,8 +76,3 @@ Family: scuemata.#Family & {
]
]
}
-
-#Latest: {
- #Dashboard: Family.latest
- #Panel: Family.latest._Panel
-}
diff --git a/pkg/cmd/grafana-server/commands/cli.go b/pkg/cmd/grafana-server/commands/cli.go
index e80b7a56803..620e2f86c4e 100644
--- a/pkg/cmd/grafana-server/commands/cli.go
+++ b/pkg/cmd/grafana-server/commands/cli.go
@@ -20,6 +20,7 @@ import (
"github.com/grafana/grafana/pkg/extensions"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
+ "github.com/grafana/grafana/pkg/infra/process"
"github.com/grafana/grafana/pkg/server"
_ "github.com/grafana/grafana/pkg/services/alerting/conditions"
_ "github.com/grafana/grafana/pkg/services/alerting/notifiers"
@@ -151,6 +152,14 @@ func executeServer(configFile, homePath, pidFile, packaging string, traceDiagnos
metrics.SetBuildInformation(opt.Version, opt.Commit, opt.BuildBranch)
+ elevated, err := process.IsRunningWithElevatedPrivileges()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error checking server process execution privilege. error: %s\n", err.Error())
+ }
+ if elevated {
+ fmt.Println("Grafana server is running with elevated privileges. This is not recommended")
+ }
+
s, err := server.Initialize(setting.CommandLineArgs{
Config: configFile, HomePath: homePath, Args: flag.Args(),
}, server.Options{
diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go
index 1fe5179cf0a..f1b4ec367aa 100644
--- a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go
+++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go
@@ -25,6 +25,7 @@ func New(cfg *setting.Cfg) *sdkhttpclient.Provider {
SetUserAgentMiddleware(userAgent),
sdkhttpclient.BasicAuthenticationMiddleware(),
sdkhttpclient.CustomHeadersMiddleware(),
+ ResponseLimitMiddleware(cfg.ResponseLimit),
}
if cfg.SigV4AuthEnabled {
diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go
index 6b6c310368b..c9a3bdd4d76 100644
--- a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go
+++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go
@@ -22,12 +22,13 @@ func TestHTTPClientProvider(t *testing.T) {
_ = New(&setting.Cfg{SigV4AuthEnabled: false})
require.Len(t, providerOpts, 1)
o := providerOpts[0]
- require.Len(t, o.Middlewares, 5)
+ require.Len(t, o.Middlewares, 6)
require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
+ require.Equal(t, ResponseLimitMiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
})
t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) {
@@ -43,12 +44,13 @@ func TestHTTPClientProvider(t *testing.T) {
_ = New(&setting.Cfg{SigV4AuthEnabled: true})
require.Len(t, providerOpts, 1)
o := providerOpts[0]
- require.Len(t, o.Middlewares, 6)
+ require.Len(t, o.Middlewares, 7)
require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
- require.Equal(t, SigV4MiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
+ require.Equal(t, ResponseLimitMiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
+ require.Equal(t, SigV4MiddlewareName, o.Middlewares[6].(sdkhttpclient.MiddlewareName).MiddlewareName())
})
}
diff --git a/pkg/infra/httpclient/httpclientprovider/response_limit_middleware.go b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware.go
new file mode 100644
index 00000000000..c17c70b3516
--- /dev/null
+++ b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware.go
@@ -0,0 +1,28 @@
+package httpclientprovider
+
+import (
+ "net/http"
+
+ sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+ "github.com/grafana/grafana/pkg/infra/httpclient"
+)
+
+// ResponseLimitMiddlewareName is the middleware name used by ResponseLimitMiddleware.
+const ResponseLimitMiddlewareName = "response-limit"
+
+func ResponseLimitMiddleware(limit int64) sdkhttpclient.Middleware {
+ return sdkhttpclient.NamedMiddlewareFunc(ResponseLimitMiddlewareName, func(opts sdkhttpclient.Options, next http.RoundTripper) http.RoundTripper {
+ if limit <= 0 {
+ return next
+ }
+ return sdkhttpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ res, err := next.RoundTrip(req)
+ if err != nil {
+ return nil, err
+ }
+
+ res.Body = httpclient.MaxBytesReader(res.Body, limit)
+ return res, nil
+ })
+ })
+}
diff --git a/pkg/infra/httpclient/httpclientprovider/response_limit_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware_test.go
new file mode 100644
index 00000000000..4a4064a8f44
--- /dev/null
+++ b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware_test.go
@@ -0,0 +1,60 @@
+package httpclientprovider
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+ "github.com/stretchr/testify/require"
+)
+
+func TestResponseLimitMiddleware(t *testing.T) {
+ tcs := []struct {
+ limit int64
+ bodyLength int
+ body string
+ err error
+ }{
+ {limit: 1, bodyLength: 1, body: "d", err: errors.New("error: http: response body too large, response limit is set to: 1")},
+ {limit: 1000000, bodyLength: 5, body: "dummy", err: nil},
+ {limit: 0, bodyLength: 5, body: "dummy", err: nil},
+ }
+ for _, tc := range tcs {
+ t.Run(fmt.Sprintf("Test ResponseLimitMiddleware with limit: %d", tc.limit), func(t *testing.T) {
+ finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: http.StatusOK, Request: req, Body: ioutil.NopCloser(strings.NewReader("dummy"))}, nil
+ })
+
+ mw := ResponseLimitMiddleware(tc.limit)
+ rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+ require.NotNil(t, rt)
+ middlewareName, ok := mw.(httpclient.MiddlewareName)
+ require.True(t, ok)
+ require.Equal(t, ResponseLimitMiddlewareName, middlewareName.MiddlewareName())
+
+ ctx := context.Background()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil)
+ require.NoError(t, err)
+ res, err := rt.RoundTrip(req)
+ require.NoError(t, err)
+ require.NotNil(t, res)
+ require.NotNil(t, res.Body)
+ require.NoError(t, res.Body.Close())
+
+ bodyBytes, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ require.EqualError(t, tc.err, err.Error())
+ } else {
+ require.NoError(t, tc.err)
+ }
+
+ require.Len(t, bodyBytes, tc.bodyLength)
+ require.Equal(t, string(bodyBytes), tc.body)
+ })
+ }
+}
diff --git a/pkg/infra/httpclient/max_bytes_reader.go b/pkg/infra/httpclient/max_bytes_reader.go
new file mode 100644
index 00000000000..9bbdce1e375
--- /dev/null
+++ b/pkg/infra/httpclient/max_bytes_reader.go
@@ -0,0 +1,66 @@
+package httpclient
+
+import (
+ "errors"
+ "fmt"
+ "io"
+)
+
+// Similar implementation to http/net MaxBytesReader
+// https://pkg.go.dev/net/http#MaxBytesReader
+// What's happening differently here, is that the field that
+// is limited is the response and not the request, thus
+// the error handling/message needed to be accurate.
+
+// ErrResponseBodyTooLarge indicates response body is too large
+var ErrResponseBodyTooLarge = errors.New("http: response body too large")
+
+// MaxBytesReader is similar to io.LimitReader but is intended for
+// limiting the size of incoming request bodies. In contrast to
+// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a
+// non-EOF error for a Read beyond the limit, and closes the
+// underlying reader when its Close method is called.
+//
+// MaxBytesReader prevents clients from accidentally or maliciously
+// sending a large request and wasting server resources.
+func MaxBytesReader(r io.ReadCloser, n int64) io.ReadCloser {
+ return &maxBytesReader{r: r, n: n}
+}
+
+type maxBytesReader struct {
+ r io.ReadCloser // underlying reader
+ n int64 // max bytes remaining
+ err error // sticky error
+}
+
+func (l *maxBytesReader) Read(p []byte) (n int, err error) {
+ if l.err != nil {
+ return 0, l.err
+ }
+ if len(p) == 0 {
+ return 0, nil
+ }
+ // If they asked for a 32KB byte read but only 5 bytes are
+ // remaining, no need to read 32KB. 6 bytes will answer the
+ // question of the whether we hit the limit or go past it.
+ if int64(len(p)) > l.n+1 {
+ p = p[:l.n+1]
+ }
+ n, err = l.r.Read(p)
+
+ if int64(n) <= l.n {
+ l.n -= int64(n)
+ l.err = err
+ return n, err
+ }
+
+ n = int(l.n)
+ l.n = 0
+
+ l.err = fmt.Errorf("error: %w, response limit is set to: %d", ErrResponseBodyTooLarge, n)
+ return n, l.err
+}
+
+func (l *maxBytesReader) Close() error {
+ return l.r.Close()
+}
diff --git a/pkg/infra/httpclient/max_bytes_reader_test.go b/pkg/infra/httpclient/max_bytes_reader_test.go
new file mode 100644
index 00000000000..13742107550
--- /dev/null
+++ b/pkg/infra/httpclient/max_bytes_reader_test.go
@@ -0,0 +1,40 @@
+package httpclient
+
+import (
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestMaxBytesReader(t *testing.T) {
+ tcs := []struct {
+ limit int64
+ bodyLength int
+ body string
+ err error
+ }{
+ {limit: 1, bodyLength: 1, body: "d", err: errors.New("error: http: response body too large, response limit is set to: 1")},
+ {limit: 1000000, bodyLength: 5, body: "dummy", err: nil},
+ {limit: 0, bodyLength: 0, body: "", err: errors.New("error: http: response body too large, response limit is set to: 0")},
+ }
+ for _, tc := range tcs {
+ t.Run(fmt.Sprintf("Test MaxBytesReader with limit: %d", tc.limit), func(t *testing.T) {
+ body := ioutil.NopCloser(strings.NewReader("dummy"))
+ readCloser := MaxBytesReader(body, tc.limit)
+
+ bodyBytes, err := ioutil.ReadAll(readCloser)
+ if err != nil {
+ require.EqualError(t, tc.err, err.Error())
+ } else {
+ require.NoError(t, tc.err)
+ }
+
+ require.Len(t, bodyBytes, tc.bodyLength)
+ require.Equal(t, string(bodyBytes), tc.body)
+ })
+ }
+}
diff --git a/pkg/infra/process/process.go b/pkg/infra/process/process.go
new file mode 100644
index 00000000000..9d2aafd8bff
--- /dev/null
+++ b/pkg/infra/process/process.go
@@ -0,0 +1,5 @@
+package process
+
+func IsRunningWithElevatedPrivileges() (bool, error) {
+ return elevatedPrivilegesCheck()
+}
diff --git a/pkg/infra/process/root_check.go b/pkg/infra/process/root_check.go
new file mode 100644
index 00000000000..bcf58a346eb
--- /dev/null
+++ b/pkg/infra/process/root_check.go
@@ -0,0 +1,20 @@
+// +build !windows
+
+package process
+
+import (
+ "fmt"
+ "os"
+ "os/user"
+)
+
+func elevatedPrivilegesCheck() (bool, error) {
+ u, err := user.Current()
+ if err != nil {
+ return false, fmt.Errorf("could not get current OS user to detect process privileges")
+ }
+
+ return (u != nil && u.Username == "root") ||
+ os.Geteuid() != os.Getuid() ||
+ os.Geteuid() == 0, nil
+}
diff --git a/pkg/infra/process/root_check_windows.go b/pkg/infra/process/root_check_windows.go
new file mode 100644
index 00000000000..41a6c1e5aab
--- /dev/null
+++ b/pkg/infra/process/root_check_windows.go
@@ -0,0 +1,8 @@
+// +build windows
+
+package process
+
+func elevatedPrivilegesCheck() (bool, error) {
+ // TODO implement Windows process root check
+ return false, nil
+}
diff --git a/pkg/infra/usagestats/service.go b/pkg/infra/usagestats/service.go
index e2171cc5733..dad282b9702 100644
--- a/pkg/infra/usagestats/service.go
+++ b/pkg/infra/usagestats/service.go
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/login/social"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/alerting"
+ "github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
)
@@ -31,17 +32,29 @@ type UsageStatsService struct {
AlertingUsageStats alerting.UsageStatsQuerier
PluginManager plugins.Manager
SocialService social.Service
+ grafanaLive *live.GrafanaLive
log log.Logger
oauthProviders map[string]bool
externalMetrics []MetricsFunc
concurrentUserStatsCache memoConcurrentUserStats
+ liveStats liveUsageStats
+}
+
+type liveUsageStats struct {
+ numClientsMax int
+ numClientsMin int
+ numClientsSum int
+ numUsersMax int
+ numUsersMin int
+ numUsersSum int
+ sampleCount int
}
func ProvideService(cfg *setting.Cfg, bus bus.Bus, sqlStore *sqlstore.SQLStore,
alertingStats alerting.UsageStatsQuerier, pluginManager plugins.Manager,
- socialService social.Service) *UsageStatsService {
+ socialService social.Service, grafanaLive *live.GrafanaLive) *UsageStatsService {
s := &UsageStatsService{
Cfg: cfg,
Bus: bus,
@@ -49,6 +62,7 @@ func ProvideService(cfg *setting.Cfg, bus bus.Bus, sqlStore *sqlstore.SQLStore,
AlertingUsageStats: alertingStats,
oauthProviders: socialService.GetOAuthProviders(),
PluginManager: pluginManager,
+ grafanaLive: grafanaLive,
log: log.New("infra.usagestats"),
}
return s
@@ -59,6 +73,7 @@ func (uss *UsageStatsService) Run(ctx context.Context) error {
sendReportTicker := time.NewTicker(time.Hour * 24)
updateStatsTicker := time.NewTicker(time.Minute * 30)
+
defer sendReportTicker.Stop()
defer updateStatsTicker.Stop()
@@ -68,8 +83,11 @@ func (uss *UsageStatsService) Run(ctx context.Context) error {
if err := uss.sendUsageStats(ctx); err != nil {
metricsLogger.Warn("Failed to send usage stats", "err", err)
}
+ // always reset live stats every report tick
+ uss.resetLiveStats()
case <-updateStatsTicker.C:
uss.updateTotalStats()
+ uss.sampleLiveStats()
case <-ctx.Done():
return ctx.Err()
}
diff --git a/pkg/infra/usagestats/usage_stats.go b/pkg/infra/usagestats/usage_stats.go
index ad2735ad249..266e4b20a23 100644
--- a/pkg/infra/usagestats/usage_stats.go
+++ b/pkg/infra/usagestats/usage_stats.go
@@ -52,6 +52,9 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards
metrics["stats.users.count"] = statsQuery.Result.Users
+ metrics["stats.admins.count"] = statsQuery.Result.Admins
+ metrics["stats.editors.count"] = statsQuery.Result.Editors
+ metrics["stats.viewers.count"] = statsQuery.Result.Viewers
metrics["stats.orgs.count"] = statsQuery.Result.Orgs
metrics["stats.playlist.count"] = statsQuery.Result.Playlists
metrics["stats.plugins.apps.count"] = uss.PluginManager.AppCount()
@@ -59,6 +62,15 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
metrics["stats.plugins.datasources.count"] = uss.PluginManager.DataSourceCount()
metrics["stats.alerts.count"] = statsQuery.Result.Alerts
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
+ metrics["stats.active_admins.count"] = statsQuery.Result.ActiveAdmins
+ metrics["stats.active_editors.count"] = statsQuery.Result.ActiveEditors
+ metrics["stats.active_viewers.count"] = statsQuery.Result.ActiveViewers
+ metrics["stats.active_sessions.count"] = statsQuery.Result.ActiveSessions
+ metrics["stats.daily_active_users.count"] = statsQuery.Result.DailyActiveUsers
+ metrics["stats.daily_active_admins.count"] = statsQuery.Result.DailyActiveAdmins
+ metrics["stats.daily_active_editors.count"] = statsQuery.Result.DailyActiveEditors
+ metrics["stats.daily_active_viewers.count"] = statsQuery.Result.DailyActiveViewers
+ metrics["stats.daily_active_sessions.count"] = statsQuery.Result.DailyActiveSessions
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
metrics["stats.stars.count"] = statsQuery.Result.Stars
metrics["stats.folders.count"] = statsQuery.Result.Folders
@@ -78,6 +90,20 @@ func (uss *UsageStatsService) GetUsageReport(ctx context.Context) (UsageReport,
metrics["stats.folders_viewers_can_edit.count"] = statsQuery.Result.FoldersViewersCanEdit
metrics["stats.folders_viewers_can_admin.count"] = statsQuery.Result.FoldersViewersCanAdmin
+ liveUsersAvg := 0
+ liveClientsAvg := 0
+ if uss.liveStats.sampleCount > 0 {
+ liveUsersAvg = uss.liveStats.numUsersSum / uss.liveStats.sampleCount
+ liveClientsAvg = uss.liveStats.numClientsSum / uss.liveStats.sampleCount
+ }
+ metrics["stats.live_samples.count"] = uss.liveStats.sampleCount
+ metrics["stats.live_users_max.count"] = uss.liveStats.numUsersMax
+ metrics["stats.live_users_min.count"] = uss.liveStats.numUsersMin
+ metrics["stats.live_users_avg.count"] = liveUsersAvg
+ metrics["stats.live_clients_max.count"] = uss.liveStats.numClientsMax
+ metrics["stats.live_clients_min.count"] = uss.liveStats.numClientsMin
+ metrics["stats.live_clients_avg.count"] = liveClientsAvg
+
ossEditionCount := 1
enterpriseEditionCount := 0
if uss.Cfg.IsEnterprise {
@@ -279,9 +305,9 @@ func (uss *UsageStatsService) sendUsageStats(ctx context.Context) error {
if err != nil {
return err
}
+
data := bytes.NewBuffer(out)
sendUsageStats(data)
-
return nil
}
@@ -302,6 +328,34 @@ var sendUsageStats = func(data *bytes.Buffer) {
}()
}
+func (uss *UsageStatsService) sampleLiveStats() {
+ current := uss.grafanaLive.UsageStats()
+
+ uss.liveStats.sampleCount++
+ uss.liveStats.numClientsSum += current.NumClients
+ uss.liveStats.numUsersSum += current.NumUsers
+
+ if current.NumClients > uss.liveStats.numClientsMax {
+ uss.liveStats.numClientsMax = current.NumClients
+ }
+
+ if current.NumClients < uss.liveStats.numClientsMin {
+ uss.liveStats.numClientsMin = current.NumClients
+ }
+
+ if current.NumUsers > uss.liveStats.numUsersMax {
+ uss.liveStats.numUsersMax = current.NumUsers
+ }
+
+ if current.NumUsers < uss.liveStats.numUsersMin {
+ uss.liveStats.numUsersMin = current.NumUsers
+ }
+}
+
+func (uss *UsageStatsService) resetLiveStats() {
+ uss.liveStats = liveUsageStats{}
+}
+
func (uss *UsageStatsService) updateTotalStats() {
if !uss.Cfg.MetricsEndpointEnabled || uss.Cfg.MetricsEndpointDisableTotalStats {
return
diff --git a/pkg/infra/usagestats/usage_stats_test.go b/pkg/infra/usagestats/usage_stats_test.go
index b521a5dbcf1..d8a950b9bea 100644
--- a/pkg/infra/usagestats/usage_stats_test.go
+++ b/pkg/infra/usagestats/usage_stats_test.go
@@ -11,12 +11,14 @@ import (
"testing"
"time"
+ "github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/services/alerting"
+ "github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
@@ -45,7 +47,19 @@ func TestMetrics(t *testing.T) {
Dashboards: 1,
Datasources: 2,
Users: 3,
+ Admins: 31,
+ Editors: 32,
+ Viewers: 33,
ActiveUsers: 4,
+ ActiveAdmins: 21,
+ ActiveEditors: 22,
+ ActiveViewers: 23,
+ ActiveSessions: 24,
+ DailyActiveUsers: 25,
+ DailyActiveAdmins: 26,
+ DailyActiveEditors: 27,
+ DailyActiveViewers: 28,
+ DailyActiveSessions: 29,
Orgs: 5,
Playlists: 6,
Alerts: 7,
@@ -296,6 +310,9 @@ func TestMetrics(t *testing.T) {
metrics := j.Get("metrics")
assert.Equal(t, getSystemStatsQuery.Result.Dashboards, metrics.Get("stats.dashboards.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Users, metrics.Get("stats.users.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.Admins, metrics.Get("stats.admins.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.Editors, metrics.Get("stats.editors.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.Viewers, metrics.Get("stats.viewers.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Orgs, metrics.Get("stats.orgs.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Playlists, metrics.Get("stats.playlist.count").MustInt64())
assert.Equal(t, uss.PluginManager.AppCount(), metrics.Get("stats.plugins.apps.count").MustInt())
@@ -303,6 +320,15 @@ func TestMetrics(t *testing.T) {
assert.Equal(t, uss.PluginManager.DataSourceCount(), metrics.Get("stats.plugins.datasources.count").MustInt())
assert.Equal(t, getSystemStatsQuery.Result.Alerts, metrics.Get("stats.alerts.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.ActiveUsers, metrics.Get("stats.active_users.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.ActiveAdmins, metrics.Get("stats.active_admins.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.ActiveEditors, metrics.Get("stats.active_editors.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.ActiveViewers, metrics.Get("stats.active_viewers.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.ActiveSessions, metrics.Get("stats.active_sessions.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.DailyActiveUsers, metrics.Get("stats.daily_active_users.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.DailyActiveAdmins, metrics.Get("stats.daily_active_admins.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.DailyActiveEditors, metrics.Get("stats.daily_active_editors.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.DailyActiveViewers, metrics.Get("stats.daily_active_viewers.count").MustInt64())
+ assert.Equal(t, getSystemStatsQuery.Result.DailyActiveSessions, metrics.Get("stats.daily_active_sessions.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Datasources, metrics.Get("stats.datasources.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Stars, metrics.Get("stats.stars.count").MustInt64())
assert.Equal(t, getSystemStatsQuery.Result.Folders, metrics.Get("stats.folders.count").MustInt64())
@@ -322,6 +348,8 @@ func TestMetrics(t *testing.T) {
assert.Equal(t, 18, metrics.Get("stats.alert_rules.count").MustInt())
assert.Equal(t, 19, metrics.Get("stats.library_panels.count").MustInt())
assert.Equal(t, 20, metrics.Get("stats.library_variables.count").MustInt())
+ assert.Equal(t, 0, metrics.Get("stats.live_users.count").MustInt())
+ assert.Equal(t, 0, metrics.Get("stats.live_clients.count").MustInt())
assert.Equal(t, 9, metrics.Get("stats.ds."+models.DS_ES+".count").MustInt())
assert.Equal(t, 10, metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt())
@@ -608,5 +636,13 @@ func createService(t *testing.T, cfg setting.Cfg) *UsageStatsService {
AlertingUsageStats: &alertingUsageMock{},
externalMetrics: make([]MetricsFunc, 0),
PluginManager: &fakePluginManager{},
+ grafanaLive: newTestLive(t),
}
}
+
+func newTestLive(t *testing.T) *live.GrafanaLive {
+ cfg := &setting.Cfg{AppURL: "http://localhost:3000/"}
+ gLive, err := live.ProvideService(nil, cfg, routing.NewRouteRegister(), nil, nil, nil, nil, sqlstore.InitTestDB(t))
+ require.NoError(t, err)
+ return gLive
+}
diff --git a/pkg/login/social/generic_oauth.go b/pkg/login/social/generic_oauth.go
index 903afaf5218..f07690d873e 100644
--- a/pkg/login/social/generic_oauth.go
+++ b/pkg/login/social/generic_oauth.go
@@ -11,6 +11,7 @@ import (
"net/http"
"net/mail"
"regexp"
+ "strconv"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/util/errutil"
@@ -21,6 +22,7 @@ type SocialGenericOAuth struct {
*SocialBase
allowedOrganizations []string
apiUrl string
+ teamsUrl string
emailAttributeName string
emailAttributePath string
loginAttributePath string
@@ -29,7 +31,8 @@ type SocialGenericOAuth struct {
roleAttributeStrict bool
groupsAttributePath string
idTokenAttributeName string
- teamIds []int
+ teamIdsAttributePath string
+ teamIds []string
}
func (s *SocialGenericOAuth) Type() int {
@@ -41,8 +44,8 @@ func (s *SocialGenericOAuth) IsTeamMember(client *http.Client) bool {
return true
}
- teamMemberships, ok := s.FetchTeamMemberships(client)
- if !ok {
+ teamMemberships, err := s.FetchTeamMemberships(client)
+ if err != nil {
return false
}
@@ -146,19 +149,21 @@ func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token)
if userInfo.Role == "" {
role, err := s.extractRole(data)
if err != nil {
- s.log.Error("Failed to extract role", "error", err)
+ s.log.Warn("Failed to extract role", "error", err)
} else if role != "" {
s.log.Debug("Setting user info role from extracted role")
userInfo.Role = role
}
}
- groups, err := s.extractGroups(data)
- if err != nil {
- s.log.Error("Failed to extract groups", "error", err)
- } else if len(groups) > 0 {
- s.log.Debug("Setting user info groups from extracted groups")
- userInfo.Groups = groups
+ if userInfo.Groups != nil && len(userInfo.Groups) == 0 {
+ groups, err := s.extractGroups(data)
+ if err != nil {
+ s.log.Warn("Failed to extract groups", "err", err)
+ } else if len(groups) > 0 {
+ s.log.Debug("Setting user info groups from extracted groups")
+ userInfo.Groups = groups
+ }
}
}
@@ -412,15 +417,36 @@ func (s *SocialGenericOAuth) FetchPrivateEmail(client *http.Client) (string, err
return email, nil
}
-func (s *SocialGenericOAuth) FetchTeamMemberships(client *http.Client) ([]int, bool) {
+func (s *SocialGenericOAuth) FetchTeamMemberships(client *http.Client) ([]string, error) {
+ var err error
+ var ids []string
+
+ if s.teamsUrl == "" {
+ ids, err = s.fetchTeamMembershipsFromDeprecatedTeamsUrl(client)
+ } else {
+ ids, err = s.fetchTeamMembershipsFromTeamsUrl(client)
+ }
+
+ if err == nil {
+ s.log.Debug("Received team memberships", "ids", ids)
+ }
+
+ return ids, err
+}
+
+func (s *SocialGenericOAuth) fetchTeamMembershipsFromDeprecatedTeamsUrl(client *http.Client) ([]string, error) {
+ var response httpGetResponse
+ var err error
+ var ids []string
+
type Record struct {
Id int `json:"id"`
}
- response, err := s.httpGet(client, fmt.Sprintf(s.apiUrl+"/teams"))
+ response, err = s.httpGet(client, fmt.Sprintf(s.apiUrl+"/teams"))
if err != nil {
s.log.Error("Error getting team memberships", "url", s.apiUrl+"/teams", "error", err)
- return nil, false
+ return []string{}, err
}
var records []Record
@@ -428,17 +454,32 @@ func (s *SocialGenericOAuth) FetchTeamMemberships(client *http.Client) ([]int, b
err = json.Unmarshal(response.Body, &records)
if err != nil {
s.log.Error("Error decoding team memberships response", "raw_json", string(response.Body), "error", err)
- return nil, false
+ return []string{}, err
}
- var ids = make([]int, len(records))
+ ids = make([]string, len(records))
for i, record := range records {
- ids[i] = record.Id
+ ids[i] = strconv.Itoa(record.Id)
}
- s.log.Debug("Received team memberships", "ids", ids)
+ return ids, nil
+}
- return ids, true
+func (s *SocialGenericOAuth) fetchTeamMembershipsFromTeamsUrl(client *http.Client) ([]string, error) {
+ if s.teamIdsAttributePath == "" {
+ return []string{}, nil
+ }
+
+ var response httpGetResponse
+ var err error
+
+ response, err = s.httpGet(client, fmt.Sprintf(s.teamsUrl))
+ if err != nil {
+ s.log.Error("Error getting team memberships", "url", s.teamsUrl, "error", err)
+ return nil, err
+ }
+
+ return s.searchJSONForStringArrayAttr(s.teamIdsAttributePath, response.Body)
}
func (s *SocialGenericOAuth) FetchOrganizations(client *http.Client) ([]string, bool) {
diff --git a/pkg/login/social/generic_oauth_test.go b/pkg/login/social/generic_oauth_test.go
index 2bfcaa51b1b..c293e1be9ad 100644
--- a/pkg/login/social/generic_oauth_test.go
+++ b/pkg/login/social/generic_oauth_test.go
@@ -379,6 +379,48 @@ func TestUserInfoSearchesForEmailAndRole(t *testing.T) {
ExpectedEmail: "john.doe@example.com",
ExpectedRole: "FromResponse",
},
+ {
+ Name: "Given a valid id_token, a valid advanced JMESPath role path, derive the role",
+ OAuth2Extra: map[string]interface{}{
+ // { "email": "john.doe@example.com",
+ // "info": { "roles": [ "dev", "engineering" ] }}
+ "id_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImpvaG4uZG9lQGV4YW1wbGUuY29tIiwiaW5mbyI6eyJyb2xlcyI6WyJkZXYiLCJlbmdpbmVlcmluZyJdfX0.RmmQfv25eXb4p3wMrJsvXfGQ6EXhGtwRXo6SlCFHRNg",
+ },
+ RoleAttributePath: "contains(info.roles[*], 'dev') && 'Editor'",
+ ExpectedEmail: "john.doe@example.com",
+ ExpectedRole: "Editor",
+ },
+ {
+ Name: "Given a valid id_token without role info, a valid advanced JMESPath role path, a valid API response, derive the correct role using the userinfo API response (JMESPath warning on id_token)",
+ OAuth2Extra: map[string]interface{}{
+ // { "email": "john.doe@example.com" }
+ "id_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImpvaG4uZG9lQGV4YW1wbGUuY29tIn0.k5GwPcZvGe2BE_jgwN0ntz0nz4KlYhEd0hRRLApkTJ4",
+ },
+ ResponseBody: map[string]interface{}{
+ "info": map[string]interface{}{
+ "roles": []string{"engineering", "SRE"},
+ },
+ },
+ RoleAttributePath: "contains(info.roles[*], 'SRE') && 'Admin'",
+ ExpectedEmail: "john.doe@example.com",
+ ExpectedRole: "Admin",
+ },
+ {
+ Name: "Given a valid id_token, a valid advanced JMESPath role path, a valid API response, prefer ID token",
+ OAuth2Extra: map[string]interface{}{
+ // { "email": "john.doe@example.com",
+ // "info": { "roles": [ "dev", "engineering" ] }}
+ "id_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImpvaG4uZG9lQGV4YW1wbGUuY29tIiwiaW5mbyI6eyJyb2xlcyI6WyJkZXYiLCJlbmdpbmVlcmluZyJdfX0.RmmQfv25eXb4p3wMrJsvXfGQ6EXhGtwRXo6SlCFHRNg",
+ },
+ ResponseBody: map[string]interface{}{
+ "info": map[string]interface{}{
+ "roles": []string{"engineering", "SRE"},
+ },
+ },
+ RoleAttributePath: "contains(info.roles[*], 'SRE') && 'Admin' || contains(info.roles[*], 'dev') && 'Editor' || 'Viewer'",
+ ExpectedEmail: "john.doe@example.com",
+ ExpectedRole: "Editor",
+ },
}
for _, test := range tests {
diff --git a/pkg/login/social/gitlab_oauth.go b/pkg/login/social/gitlab_oauth.go
index afb8731c629..17bd231e1a4 100644
--- a/pkg/login/social/gitlab_oauth.go
+++ b/pkg/login/social/gitlab_oauth.go
@@ -13,8 +13,9 @@ import (
type SocialGitlab struct {
*SocialBase
- allowedGroups []string
- apiUrl string
+ allowedGroups []string
+ apiUrl string
+ roleAttributePath string
}
func (s *SocialGitlab) Type() int {
@@ -114,12 +115,18 @@ func (s *SocialGitlab) UserInfo(client *http.Client, token *oauth2.Token) (*Basi
groups := s.GetGroups(client)
+ role, err := s.extractRole(response.Body)
+ if err != nil {
+ s.log.Error("Failed to extract role", "error", err)
+ }
+
userInfo := &BasicUserInfo{
Id: fmt.Sprintf("%d", data.Id),
Name: data.Name,
Login: data.Username,
Email: data.Email,
Groups: groups,
+ Role: role,
}
if !s.IsGroupMember(groups) {
@@ -128,3 +135,16 @@ func (s *SocialGitlab) UserInfo(client *http.Client, token *oauth2.Token) (*Basi
return userInfo, nil
}
+
+func (s *SocialGitlab) extractRole(rawJSON []byte) (string, error) {
+ if s.roleAttributePath == "" {
+ return "", nil
+ }
+
+ role, err := s.searchJSONForStringAttr(s.roleAttributePath, rawJSON)
+
+ if err != nil {
+ return "", err
+ }
+ return role, nil
+}
diff --git a/pkg/login/social/social.go b/pkg/login/social/social.go
index 2c07aa2384b..948d4b1a72b 100644
--- a/pkg/login/social/social.go
+++ b/pkg/login/social/social.go
@@ -38,9 +38,11 @@ type OAuthInfo struct {
RoleAttributePath string
RoleAttributeStrict bool
GroupsAttributePath string
+ TeamIdsAttributePath string
AllowedDomains []string
HostedDomain string
ApiUrl string
+ TeamsUrl string
AllowSignup bool
Name string
TlsClientCert string
@@ -60,26 +62,28 @@ func ProvideService(cfg *setting.Cfg) *SocialService {
sec := cfg.Raw.Section("auth." + name)
info := &OAuthInfo{
- ClientId: sec.Key("client_id").String(),
- ClientSecret: sec.Key("client_secret").String(),
- Scopes: util.SplitString(sec.Key("scopes").String()),
- AuthUrl: sec.Key("auth_url").String(),
- TokenUrl: sec.Key("token_url").String(),
- ApiUrl: sec.Key("api_url").String(),
- Enabled: sec.Key("enabled").MustBool(),
- EmailAttributeName: sec.Key("email_attribute_name").String(),
- EmailAttributePath: sec.Key("email_attribute_path").String(),
- RoleAttributePath: sec.Key("role_attribute_path").String(),
- RoleAttributeStrict: sec.Key("role_attribute_strict").MustBool(),
- GroupsAttributePath: sec.Key("groups_attribute_path").String(),
- AllowedDomains: util.SplitString(sec.Key("allowed_domains").String()),
- HostedDomain: sec.Key("hosted_domain").String(),
- AllowSignup: sec.Key("allow_sign_up").MustBool(),
- Name: sec.Key("name").MustString(name),
- TlsClientCert: sec.Key("tls_client_cert").String(),
- TlsClientKey: sec.Key("tls_client_key").String(),
- TlsClientCa: sec.Key("tls_client_ca").String(),
- TlsSkipVerify: sec.Key("tls_skip_verify_insecure").MustBool(),
+ ClientId: sec.Key("client_id").String(),
+ ClientSecret: sec.Key("client_secret").String(),
+ Scopes: util.SplitString(sec.Key("scopes").String()),
+ AuthUrl: sec.Key("auth_url").String(),
+ TokenUrl: sec.Key("token_url").String(),
+ ApiUrl: sec.Key("api_url").String(),
+ TeamsUrl: sec.Key("teams_url").String(),
+ Enabled: sec.Key("enabled").MustBool(),
+ EmailAttributeName: sec.Key("email_attribute_name").String(),
+ EmailAttributePath: sec.Key("email_attribute_path").String(),
+ RoleAttributePath: sec.Key("role_attribute_path").String(),
+ RoleAttributeStrict: sec.Key("role_attribute_strict").MustBool(),
+ GroupsAttributePath: sec.Key("groups_attribute_path").String(),
+ TeamIdsAttributePath: sec.Key("team_ids_attribute_path").String(),
+ AllowedDomains: util.SplitString(sec.Key("allowed_domains").String()),
+ HostedDomain: sec.Key("hosted_domain").String(),
+ AllowSignup: sec.Key("allow_sign_up").MustBool(),
+ Name: sec.Key("name").MustString(name),
+ TlsClientCert: sec.Key("tls_client_cert").String(),
+ TlsClientKey: sec.Key("tls_client_key").String(),
+ TlsClientCa: sec.Key("tls_client_ca").String(),
+ TlsSkipVerify: sec.Key("tls_skip_verify_insecure").MustBool(),
}
// when empty_scopes parameter exists and is true, overwrite scope with empty value
@@ -122,9 +126,10 @@ func ProvideService(cfg *setting.Cfg) *SocialService {
// GitLab.
if name == "gitlab" {
ss.socialMap["gitlab"] = &SocialGitlab{
- SocialBase: newSocialBase(name, &config, info),
- apiUrl: info.ApiUrl,
- allowedGroups: util.SplitString(sec.Key("allowed_groups").String()),
+ SocialBase: newSocialBase(name, &config, info),
+ apiUrl: info.ApiUrl,
+ allowedGroups: util.SplitString(sec.Key("allowed_groups").String()),
+ roleAttributePath: info.RoleAttributePath,
}
}
@@ -162,6 +167,7 @@ func ProvideService(cfg *setting.Cfg) *SocialService {
ss.socialMap["generic_oauth"] = &SocialGenericOAuth{
SocialBase: newSocialBase(name, &config, info),
apiUrl: info.ApiUrl,
+ teamsUrl: info.TeamsUrl,
emailAttributeName: info.EmailAttributeName,
emailAttributePath: info.EmailAttributePath,
nameAttributePath: sec.Key("name_attribute_path").String(),
@@ -170,7 +176,8 @@ func ProvideService(cfg *setting.Cfg) *SocialService {
groupsAttributePath: info.GroupsAttributePath,
loginAttributePath: sec.Key("login_attribute_path").String(),
idTokenAttributeName: sec.Key("id_token_attribute_name").String(),
- teamIds: sec.Key("team_ids").Ints(","),
+ teamIdsAttributePath: sec.Key("team_ids_attribute_path").String(),
+ teamIds: sec.Key("team_ids").Strings(","),
allowedOrganizations: util.SplitString(sec.Key("allowed_organizations").String()),
}
}
diff --git a/pkg/macaron/binding/binding.go b/pkg/macaron/binding/binding.go
index 9ba5496d9e0..293bee9fca1 100644
--- a/pkg/macaron/binding/binding.go
+++ b/pkg/macaron/binding/binding.go
@@ -419,6 +419,10 @@ func validateField(errors Errors, zero interface{}, field reflect.StructField, f
sliceVal = sliceVal.Elem()
}
+ if sliceVal.Kind() == reflect.Invalid {
+ continue
+ }
+
sliceValue := sliceVal.Interface()
zero := reflect.Zero(sliceVal.Type()).Interface()
if sliceVal.Kind() == reflect.Struct ||
diff --git a/pkg/macaron/context.go b/pkg/macaron/context.go
index d3e917bfdab..1c1164e58d6 100644
--- a/pkg/macaron/context.go
+++ b/pkg/macaron/context.go
@@ -45,7 +45,6 @@ type Context struct {
Resp ResponseWriter
params Params
template *template.Template
- Data map[string]interface{}
}
func (ctx *Context) handler() Handler {
@@ -64,27 +63,13 @@ func (ctx *Context) Next() {
ctx.run()
}
-// Written returns whether the context response has been written to
-func (ctx *Context) Written() bool {
- return ctx.Resp.Written()
-}
-
func (ctx *Context) run() {
for ctx.index <= len(ctx.handlers) {
- vals, err := ctx.Invoke(ctx.handler())
- if err != nil {
+ if _, err := ctx.Invoke(ctx.handler()); err != nil {
panic(err)
}
ctx.index++
-
- // if the handler returned something, write it to the http response
- if len(vals) > 0 {
- ev := ctx.GetVal(reflect.TypeOf(ReturnHandler(nil)))
- handleReturn := ev.Interface().(ReturnHandler)
- handleReturn(ctx, vals)
- }
-
- if ctx.Written() {
+ if ctx.Resp.Written() {
return
}
}
diff --git a/pkg/macaron/macaron.go b/pkg/macaron/macaron.go
index d6f0c4379fd..714445e5720 100644
--- a/pkg/macaron/macaron.go
+++ b/pkg/macaron/macaron.go
@@ -157,10 +157,8 @@ func (m *Macaron) createContext(rw http.ResponseWriter, req *http.Request) *Cont
index: 0,
Router: m.Router,
Resp: NewResponseWriter(req.Method, rw),
- Data: make(map[string]interface{}),
}
req = req.WithContext(context.WithValue(req.Context(), macaronContextKey{}, c))
- c.Map(defaultReturnHandler())
c.Map(c)
c.MapTo(c.Resp, (*http.ResponseWriter)(nil))
c.Map(req)
diff --git a/pkg/macaron/return_handler.go b/pkg/macaron/return_handler.go
deleted file mode 100644
index efef439dd97..00000000000
--- a/pkg/macaron/return_handler.go
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright 2013 Martini Authors
-// Copyright 2014 The Macaron Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"): you may
-// not use this file except in compliance with the License. You may obtain
-// a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations
-// under the License.
-
-package macaron
-
-import (
- "net/http"
- "reflect"
-)
-
-// ReturnHandler is a service that Martini provides that is called
-// when a route handler returns something. The ReturnHandler is
-// responsible for writing to the ResponseWriter based on the values
-// that are passed into this function.
-type ReturnHandler func(*Context, []reflect.Value)
-
-func canDeref(val reflect.Value) bool {
- return val.Kind() == reflect.Interface || val.Kind() == reflect.Ptr
-}
-
-func isError(val reflect.Value) bool {
- _, ok := val.Interface().(error)
- return ok
-}
-
-func isByteSlice(val reflect.Value) bool {
- return val.Kind() == reflect.Slice && val.Type().Elem().Kind() == reflect.Uint8
-}
-
-func defaultReturnHandler() ReturnHandler {
- return func(ctx *Context, vals []reflect.Value) {
- rv := ctx.GetVal(InterfaceOf((*http.ResponseWriter)(nil)))
- resp := rv.Interface().(http.ResponseWriter)
- var respVal reflect.Value
- if len(vals) > 1 && vals[0].Kind() == reflect.Int {
- resp.WriteHeader(int(vals[0].Int()))
- respVal = vals[1]
- } else if len(vals) > 0 {
- respVal = vals[0]
-
- if isError(respVal) {
- err := respVal.Interface().(error)
- if err != nil {
- http.Error(resp, err.Error(), 500)
- }
- return
- } else if canDeref(respVal) {
- if respVal.IsNil() {
- return // Ignore nil error
- }
- }
- }
- if canDeref(respVal) {
- respVal = respVal.Elem()
- }
- if isByteSlice(respVal) {
- _, _ = resp.Write(respVal.Bytes())
- } else {
- _, _ = resp.Write([]byte(respVal.String()))
- }
- }
-}
diff --git a/pkg/middleware/auth.go b/pkg/middleware/auth.go
index 267a69c87a6..ff9b66433d9 100644
--- a/pkg/middleware/auth.go
+++ b/pkg/middleware/auth.go
@@ -111,9 +111,8 @@ func Auth(options *AuthOptions) macaron.Handler {
requireLogin := !c.AllowAnonymous || forceLogin || options.ReqNoAnonynmous
if !c.IsSignedIn && options.ReqSignedIn && requireLogin {
- lookupTokenErr, hasTokenErr := c.Data["lookupTokenErr"].(error)
var revokedErr *models.TokenRevokedError
- if hasTokenErr && errors.As(lookupTokenErr, &revokedErr) {
+ if errors.As(c.LookupTokenErr, &revokedErr) {
tokenRevoked(c, revokedErr)
return
}
diff --git a/pkg/middleware/logger.go b/pkg/middleware/logger.go
index d9e1daa411a..9732ea2c7c2 100644
--- a/pkg/middleware/logger.go
+++ b/pkg/middleware/logger.go
@@ -19,9 +19,8 @@ import (
"net/http"
"time"
- "github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/services/contexthandler"
"github.com/grafana/grafana/pkg/setting"
- "github.com/prometheus/client_golang/prometheus"
cw "github.com/weaveworks/common/middleware"
"gopkg.in/macaron.v1"
)
@@ -29,16 +28,15 @@ import (
func Logger(cfg *setting.Cfg) macaron.Handler {
return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
start := time.Now()
- c.Data["perfmon.start"] = start
rw := res.(macaron.ResponseWriter)
c.Next()
timeTaken := time.Since(start) / time.Millisecond
- if timer, ok := c.Data["perfmon.timer"]; ok {
- timerTyped := timer.(prometheus.Summary)
- timerTyped.Observe(float64(timeTaken))
+ ctx := contexthandler.FromContext(c.Req.Context())
+ if ctx != nil && ctx.PerfmonTimer != nil {
+ ctx.PerfmonTimer.Observe(float64(timeTaken))
}
status := rw.Status()
@@ -48,9 +46,7 @@ func Logger(cfg *setting.Cfg) macaron.Handler {
}
}
- if ctx, ok := c.Data["ctx"]; ok {
- ctxTyped := ctx.(*models.ReqContext)
-
+ if ctx != nil {
logParams := []interface{}{
"method", req.Method,
"path", req.URL.Path,
@@ -61,15 +57,15 @@ func Logger(cfg *setting.Cfg) macaron.Handler {
"referer", req.Referer(),
}
- traceID, exist := cw.ExtractTraceID(ctxTyped.Req.Context())
+ traceID, exist := cw.ExtractTraceID(ctx.Req.Context())
if exist {
logParams = append(logParams, "traceID", traceID)
}
if status >= 500 {
- ctxTyped.Logger.Error("Request Completed", logParams...)
+ ctx.Logger.Error("Request Completed", logParams...)
} else {
- ctxTyped.Logger.Info("Request Completed", logParams...)
+ ctx.Logger.Info("Request Completed", logParams...)
}
}
}
diff --git a/pkg/middleware/org_redirect.go b/pkg/middleware/org_redirect.go
index 59cbddf58e9..8dfc87a7edf 100644
--- a/pkg/middleware/org_redirect.go
+++ b/pkg/middleware/org_redirect.go
@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/services/contexthandler"
"github.com/grafana/grafana/pkg/setting"
"gopkg.in/macaron.v1"
)
@@ -23,8 +24,8 @@ func OrgRedirect(cfg *setting.Cfg) macaron.Handler {
return
}
- ctx, ok := c.Data["ctx"].(*models.ReqContext)
- if !ok || !ctx.IsSignedIn {
+ ctx := contexthandler.FromContext(req.Context())
+ if !ctx.IsSignedIn {
return
}
diff --git a/pkg/middleware/recovery.go b/pkg/middleware/recovery.go
index 88a4f8b32e0..5fbad323a92 100644
--- a/pkg/middleware/recovery.go
+++ b/pkg/middleware/recovery.go
@@ -26,7 +26,7 @@ import (
"gopkg.in/macaron.v1"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/services/contexthandler"
"github.com/grafana/grafana/pkg/setting"
)
@@ -109,9 +109,9 @@ func Recovery(cfg *setting.Cfg) macaron.Handler {
if r := recover(); r != nil {
panicLogger := log.Root
// try to get request logger
- if ctx, ok := c.Data["ctx"]; ok {
- ctxTyped := ctx.(*models.ReqContext)
- panicLogger = ctxTyped.Logger
+ ctx := contexthandler.FromContext(c.Req.Context())
+ if ctx != nil {
+ panicLogger = ctx.Logger
}
if err, ok := r.(error); ok {
@@ -128,37 +128,38 @@ func Recovery(cfg *setting.Cfg) macaron.Handler {
panicLogger.Error("Request error", "error", r, "stack", string(stack))
// if response has already been written, skip.
- if c.Written() {
+ if c.Resp.Written() {
return
}
- c.Data["Title"] = "Server Error"
- c.Data["AppSubUrl"] = cfg.AppSubURL
- c.Data["Theme"] = cfg.DefaultTheme
+ data := struct {
+ Title string
+ AppSubUrl string
+ Theme string
+ ErrorMsg string
+ }{"Server Error", cfg.AppSubURL, cfg.DefaultTheme, ""}
if setting.Env == setting.Dev {
if err, ok := r.(error); ok {
- c.Data["Title"] = err.Error()
+ data.Title = err.Error()
}
- c.Data["ErrorMsg"] = string(stack)
+ data.ErrorMsg = string(stack)
}
- ctx, ok := c.Data["ctx"].(*models.ReqContext)
-
- if ok && ctx.IsApiRequest() {
+ if ctx != nil && ctx.IsApiRequest() {
resp := make(map[string]interface{})
resp["message"] = "Internal Server Error - Check the Grafana server logs for the detailed error message."
- if c.Data["ErrorMsg"] != nil {
- resp["error"] = fmt.Sprintf("%v - %v", c.Data["Title"], c.Data["ErrorMsg"])
+ if data.ErrorMsg != "" {
+ resp["error"] = fmt.Sprintf("%v - %v", data.Title, data.ErrorMsg)
} else {
- resp["error"] = c.Data["Title"]
+ resp["error"] = data.Title
}
c.JSON(500, resp)
} else {
- c.HTML(500, cfg.ErrTemplateName, c.Data)
+ c.HTML(500, cfg.ErrTemplateName, data)
}
}
}()
diff --git a/pkg/middleware/request_tracing.go b/pkg/middleware/request_tracing.go
index 32e44097dae..d2a9b5b9bc8 100644
--- a/pkg/middleware/request_tracing.go
+++ b/pkg/middleware/request_tracing.go
@@ -52,6 +52,7 @@ func RequestTracing() macaron.Handler {
ctx := opentracing.ContextWithSpan(req.Context(), span)
c.Req = req.WithContext(ctx)
+ c.Map(c.Req)
c.Next()
diff --git a/pkg/models/context.go b/pkg/models/context.go
index a3bb6838985..b631f80bc22 100644
--- a/pkg/models/context.go
+++ b/pkg/models/context.go
@@ -21,22 +21,27 @@ type ReqContext struct {
Logger log.Logger
// RequestNonce is a cryptographic request identifier for use with Content Security Policy.
RequestNonce string
+
+ PerfmonTimer prometheus.Summary
+ LookupTokenErr error
}
// Handle handles and logs error by given status.
func (ctx *ReqContext) Handle(cfg *setting.Cfg, status int, title string, err error) {
+ data := struct {
+ Title string
+ AppSubUrl string
+ Theme string
+ ErrorMsg error
+ }{title, cfg.AppSubURL, "dark", nil}
if err != nil {
ctx.Logger.Error(title, "error", err)
if setting.Env != setting.Prod {
- ctx.Data["ErrorMsg"] = err
+ data.ErrorMsg = err
}
}
- ctx.Data["Title"] = title
- ctx.Data["AppSubUrl"] = cfg.AppSubURL
- ctx.Data["Theme"] = "dark"
-
- ctx.HTML(status, cfg.ErrTemplateName, ctx.Data)
+ ctx.HTML(status, cfg.ErrTemplateName, data)
}
func (ctx *ReqContext) IsApiRequest() bool {
@@ -76,7 +81,7 @@ func (ctx *ReqContext) HasHelpFlag(flag HelpFlags1) bool {
}
func (ctx *ReqContext) TimeRequest(timer prometheus.Summary) {
- ctx.Data["perfmon.timer"] = timer
+ ctx.PerfmonTimer = timer
}
// QueryBoolWithDefault extracts a value from the request query params and applies a bool default if not present.
diff --git a/pkg/models/stats.go b/pkg/models/stats.go
index 9036d9ca5da..43bb80bdeec 100644
--- a/pkg/models/stats.go
+++ b/pkg/models/stats.go
@@ -5,6 +5,7 @@ type SystemStats struct {
Datasources int64
Users int64
ActiveUsers int64
+ DailyActiveUsers int64
Orgs int64
Playlists int64
Alerts int64
@@ -25,14 +26,17 @@ type SystemStats struct {
DashboardsViewersCanAdmin int64
FoldersViewersCanEdit int64
FoldersViewersCanAdmin int64
-
- Admins int
- Editors int
- Viewers int
- ActiveAdmins int
- ActiveEditors int
- ActiveViewers int
- ActiveSessions int
+ Admins int64
+ Editors int64
+ Viewers int64
+ ActiveAdmins int64
+ ActiveEditors int64
+ ActiveViewers int64
+ ActiveSessions int64
+ DailyActiveAdmins int64
+ DailyActiveEditors int64
+ DailyActiveViewers int64
+ DailyActiveSessions int64
}
type DataSourceStats struct {
@@ -68,23 +72,28 @@ type GetAlertNotifierUsageStatsQuery struct {
}
type AdminStats struct {
- Orgs int `json:"orgs"`
- Dashboards int `json:"dashboards"`
- Snapshots int `json:"snapshots"`
- Tags int `json:"tags"`
- Datasources int `json:"datasources"`
- Playlists int `json:"playlists"`
- Stars int `json:"stars"`
- Alerts int `json:"alerts"`
- Users int `json:"users"`
- Admins int `json:"admins"`
- Editors int `json:"editors"`
- Viewers int `json:"viewers"`
- ActiveUsers int `json:"activeUsers"`
- ActiveAdmins int `json:"activeAdmins"`
- ActiveEditors int `json:"activeEditors"`
- ActiveViewers int `json:"activeViewers"`
- ActiveSessions int `json:"activeSessions"`
+ Orgs int64 `json:"orgs"`
+ Dashboards int64 `json:"dashboards"`
+ Snapshots int64 `json:"snapshots"`
+ Tags int64 `json:"tags"`
+ Datasources int64 `json:"datasources"`
+ Playlists int64 `json:"playlists"`
+ Stars int64 `json:"stars"`
+ Alerts int64 `json:"alerts"`
+ Users int64 `json:"users"`
+ Admins int64 `json:"admins"`
+ Editors int64 `json:"editors"`
+ Viewers int64 `json:"viewers"`
+ ActiveUsers int64 `json:"activeUsers"`
+ ActiveAdmins int64 `json:"activeAdmins"`
+ ActiveEditors int64 `json:"activeEditors"`
+ ActiveViewers int64 `json:"activeViewers"`
+ ActiveSessions int64 `json:"activeSessions"`
+ DailyActiveUsers int64 `json:"dailyActiveUsers"`
+ DailyActiveAdmins int64 `json:"dailyActiveAdmins"`
+ DailyActiveEditors int64 `json:"dailyActiveEditors"`
+ DailyActiveViewers int64 `json:"dailyActiveViewers"`
+ DailyActiveSessions int64 `json:"dailyActiveSessions"`
}
type GetAdminStatsQuery struct {
diff --git a/pkg/models/test_data.go b/pkg/models/test_data.go
deleted file mode 100644
index 1ed34dce99b..00000000000
--- a/pkg/models/test_data.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package models
-
-import "time"
-
-type InsertSQLTestDataCommand struct {
-}
-
-type SQLTestData struct {
- Id int64
- Metric1 string
- Metric2 string
- ValueBigInt int64
- ValueDouble float64
- ValueFloat float32
- ValueInt int
- TimeEpoch int64
- TimeDateTime time.Time
-}
diff --git a/pkg/plugins/backendplugin/grpcplugin/grpc_plugin.go b/pkg/plugins/backendplugin/grpcplugin/grpc_plugin.go
index 47231f2c7b2..a2af94132aa 100644
--- a/pkg/plugins/backendplugin/grpcplugin/grpc_plugin.go
+++ b/pkg/plugins/backendplugin/grpcplugin/grpc_plugin.go
@@ -7,6 +7,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
+ "github.com/grafana/grafana/pkg/infra/process"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/hashicorp/go-plugin"
)
@@ -72,6 +73,14 @@ func (p *grpcPlugin) Start(ctx context.Context) error {
return errors.New("no compatible plugin implementation found")
}
+ elevated, err := process.IsRunningWithElevatedPrivileges()
+ if err != nil {
+ p.logger.Debug("Error checking plugin process execution privilege", "err", err)
+ }
+ if elevated {
+ p.logger.Warn("Plugin process is running with elevated privileges. This is not recommended")
+ }
+
return nil
}
diff --git a/pkg/plugins/manager/manager.go b/pkg/plugins/manager/manager.go
index 7a4699dc9f5..3efbb2bdfbe 100644
--- a/pkg/plugins/manager/manager.go
+++ b/pkg/plugins/manager/manager.go
@@ -512,6 +512,7 @@ func (pm *PluginManager) loadPlugin(jsonParser *json.Decoder, pluginBase *plugin
pb.Signature = pluginBase.Signature
pb.SignatureType = pluginBase.SignatureType
pb.SignatureOrg = pluginBase.SignatureOrg
+ pb.SignedFiles = pluginBase.SignedFiles
pm.plugins[pb.Id] = pb
pm.log.Debug("Successfully added plugin", "id", pb.Id)
@@ -581,6 +582,7 @@ func (s *PluginScanner) loadPlugin(pluginJSONFilePath string) error {
pluginCommon.Signature = signatureState.Status
pluginCommon.SignatureType = signatureState.Type
pluginCommon.SignatureOrg = signatureState.SigningOrg
+ pluginCommon.SignedFiles = signatureState.Files
s.plugins[currentDir] = &pluginCommon
diff --git a/pkg/plugins/manager/manager_test.go b/pkg/plugins/manager/manager_test.go
index 64bef480aa8..32121dd6877 100644
--- a/pkg/plugins/manager/manager_test.go
+++ b/pkg/plugins/manager/manager_test.go
@@ -232,6 +232,7 @@ func TestPluginManager_Init(t *testing.T) {
Signature: plugins.PluginSignatureValid,
SignatureType: plugins.GrafanaType,
SignatureOrg: "Grafana Labs",
+ SignedFiles: plugins.PluginFiles{"plugin.json": {}},
Dependencies: plugins.PluginDependencies{
GrafanaVersion: "*",
Plugins: []plugins.PluginDependencyItem{},
@@ -497,6 +498,7 @@ func TestPluginManager_Installer(t *testing.T) {
Signature: plugins.PluginSignatureValid,
SignatureType: plugins.GrafanaType,
SignatureOrg: "Grafana Labs",
+ SignedFiles: plugins.PluginFiles{"plugin.json": {}},
Dependencies: plugins.PluginDependencies{
GrafanaVersion: "*",
Plugins: []plugins.PluginDependencyItem{},
diff --git a/pkg/plugins/manager/manifest.go b/pkg/plugins/manager/manifest.go
index 1d5118c1d55..58b03558bd6 100644
--- a/pkg/plugins/manager/manifest.go
+++ b/pkg/plugins/manager/manifest.go
@@ -169,45 +169,19 @@ func getPluginSignatureState(log log.Logger, plugin *plugins.PluginBase) (plugin
}
}
- manifestFiles := make(map[string]bool, len(manifest.Files))
+ manifestFiles := make(map[string]struct{}, len(manifest.Files))
// Verify the manifest contents
log.Debug("Verifying contents of plugin manifest", "plugin", plugin.Id)
- for p, hash := range manifest.Files {
- // Open the file
- fp := filepath.Join(plugin.PluginDir, p)
-
- // nolint:gosec
- // We can ignore the gosec G304 warning on this one because `fp` is based
- // on the manifest file for a plugin and not user input.
- f, err := os.Open(fp)
+ for fp, hash := range manifest.Files {
+ err = verifyHash(plugin.Id, filepath.Join(plugin.PluginDir, fp), hash)
if err != nil {
- log.Warn("Plugin file listed in the manifest was not found", "plugin", plugin.Id, "filename", p, "dir", plugin.PluginDir)
return plugins.PluginSignatureState{
Status: plugins.PluginSignatureModified,
}, nil
}
- defer func() {
- if err := f.Close(); err != nil {
- log.Warn("Failed to close plugin file", "path", fp, "err", err)
- }
- }()
- h := sha256.New()
- if _, err := io.Copy(h, f); err != nil {
- log.Warn("Couldn't read plugin file", "plugin", plugin.Id, "filename", fp)
- return plugins.PluginSignatureState{
- Status: plugins.PluginSignatureModified,
- }, nil
- }
- sum := hex.EncodeToString(h.Sum(nil))
- if sum != hash {
- log.Warn("Plugin file's signature has been modified versus manifest", "plugin", plugin.Id, "filename", fp)
- return plugins.PluginSignatureState{
- Status: plugins.PluginSignatureModified,
- }, nil
- }
- manifestFiles[p] = true
+ manifestFiles[fp] = struct{}{}
}
if manifest.isV2() {
@@ -241,9 +215,38 @@ func getPluginSignatureState(log log.Logger, plugin *plugins.PluginBase) (plugin
Status: plugins.PluginSignatureValid,
Type: manifest.SignatureType,
SigningOrg: manifest.SignedByOrgName,
+ Files: manifestFiles,
}, nil
}
+func verifyHash(pluginID string, path string, hash string) error {
+ // nolint:gosec
+ // We can ignore the gosec G304 warning on this one because `path` is based
+ // on the path provided in a manifest file for a plugin and not user input.
+ f, err := os.Open(path)
+ if err != nil {
+ log.Warn("Plugin file listed in the manifest was not found", "plugin", pluginID, "path", path)
+ return fmt.Errorf("plugin file listed in the manifest was not found")
+ }
+ defer func() {
+ if err := f.Close(); err != nil {
+ log.Warn("Failed to close plugin file", "path", path, "err", err)
+ }
+ }()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return fmt.Errorf("could not calculate plugin file checksum")
+ }
+ sum := hex.EncodeToString(h.Sum(nil))
+ if sum != hash {
+ log.Warn("Plugin file checksum does not match signature checksum", "plugin", pluginID, "path", path)
+ return fmt.Errorf("plugin file checksum does not match signature checksum")
+ }
+
+ return nil
+}
+
// gets plugin filenames that require verification for plugin signing
// returns filenames as a slice of posix style paths relative to plugin directory
func pluginFilesRequiringVerification(plugin *plugins.PluginBase) ([]string, error) {
diff --git a/pkg/plugins/models.go b/pkg/plugins/models.go
index 414ad9b0b77..613ee44b2a5 100644
--- a/pkg/plugins/models.go
+++ b/pkg/plugins/models.go
@@ -73,6 +73,7 @@ type PluginBase struct {
IsCorePlugin bool `json:"-"`
SignatureType PluginSignatureType `json:"-"`
SignatureOrg string `json:"-"`
+ SignedFiles PluginFiles `json:"-"`
GrafanaNetVersion string `json:"-"`
GrafanaNetHasUpdate bool `json:"-"`
@@ -80,6 +81,23 @@ type PluginBase struct {
Root *PluginBase
}
+func (p *PluginBase) IncludedInSignature(file string) bool {
+ // permit Core plugin files
+ if p.IsCorePlugin {
+ return true
+ }
+
+ // permit when no signed files (no MANIFEST)
+ if p.SignedFiles == nil {
+ return true
+ }
+
+ if _, exists := p.SignedFiles[file]; !exists {
+ return false
+ }
+ return true
+}
+
type PluginDependencies struct {
GrafanaVersion string `json:"grafanaVersion"`
Plugins []PluginDependencyItem `json:"plugins"`
diff --git a/pkg/plugins/state.go b/pkg/plugins/state.go
index 313b6ec3c83..3cf2b879123 100644
--- a/pkg/plugins/state.go
+++ b/pkg/plugins/state.go
@@ -31,8 +31,11 @@ const (
PrivateType PluginSignatureType = "private"
)
+type PluginFiles map[string]struct{}
+
type PluginSignatureState struct {
Status PluginSignatureStatus
Type PluginSignatureType
SigningOrg string
+ Files PluginFiles
}
diff --git a/pkg/schema/load/dashboard.go b/pkg/schema/load/dashboard.go
index 302080e3a2c..71aceb9cacf 100644
--- a/pkg/schema/load/dashboard.go
+++ b/pkg/schema/load/dashboard.go
@@ -36,10 +36,19 @@ func defaultOverlay(p BaseLoadPaths) (map[string]load.Source, error) {
// family: the 0.0 schema. schema.Find() provides easy traversal to newer schema
// versions.
func BaseDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
- overlay, err := defaultOverlay(p)
+ v, err := baseDashboardFamily(p)
if err != nil {
return nil, err
}
+ return buildGenericScuemata(v)
+}
+
+// Helper that gets the entire scuemata family, for reuse by Dist/Instance callers.
+func baseDashboardFamily(p BaseLoadPaths) (cue.Value, error) {
+ overlay, err := defaultOverlay(p)
+ if err != nil {
+ return cue.Value{}, err
+ }
cfg := &load.Config{
Overlay: overlay,
@@ -51,16 +60,16 @@ func BaseDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
if err != nil {
cueError := schema.WrapCUEError(err)
if err != nil {
- return nil, cueError
+ return cue.Value{}, cueError
}
}
famval := inst.Value().LookupPath(cue.MakePath(cue.Str("Family")))
if !famval.Exists() {
- return nil, errors.New("dashboard schema family did not exist at expected path in expected file")
+ return cue.Value{}, errors.New("dashboard schema family did not exist at expected path in expected file")
}
- return buildGenericScuemata(famval)
+ return famval, nil
}
// DistDashboardFamily loads the family of schema representing the "Dist"
@@ -73,38 +82,41 @@ func BaseDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
// family: the 0.0 schema. schema.Find() provides easy traversal to newer schema
// versions.
func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
- head, err := BaseDashboardFamily(p)
+ famval, err := baseDashboardFamily(p)
if err != nil {
return nil, err
}
- scuemap, err := readPanelModels(p)
+ scuemap, err := loadPanelScuemata(p)
if err != nil {
return nil, err
}
- dj, err := disjunctPanelScuemata(scuemap)
- if err != nil {
- return nil, err
- }
- // Stick this into a dummy struct so that we can unify it into place, as
- // Value.Fill() can't target definitions. Need new method based on cue.Path;
- // a CL has been merged that creates FillPath and will be in the next
- // release of CUE.
- dummy, _ := rt.Compile("glue-unifyPanelDashboard", `
- obj: {}
- dummy: {
- #Panel: obj
- }
- `)
- filled := dummy.Value().FillPath(cue.MakePath(cue.Str("obj")), dj)
- ddj := filled.LookupPath(cue.MakePath(cue.Str("dummy")))
+ // TODO see if unifying into the expected form in a loop, then unifying that
+ // consolidated form improves performance
+ for typ, fam := range scuemap {
+ famval = famval.FillPath(cue.MakePath(cue.Str("compose"), cue.Str("Panel"), cue.Str(typ)), fam)
+ }
+ head, err := buildGenericScuemata(famval)
+ if err != nil {
+ return nil, err
+ }
+
+ // TODO sloppy duplicate logic of what's in readPanelModels(), for now
+ all := make(map[string]schema.VersionedCueSchema)
+ for id, val := range scuemap {
+ fam, err := buildGenericScuemata(val)
+ if err != nil {
+ return nil, err
+ }
+ all[id] = fam
+ }
var first, prev *compositeDashboardSchema
for head != nil {
cds := &compositeDashboardSchema{
base: head,
- actual: head.CUE().Unify(ddj),
- panelFams: scuemap,
+ actual: head.CUE(),
+ panelFams: all,
// TODO migrations
migration: terminalMigrationFunc,
}
@@ -118,7 +130,6 @@ func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
prev = cds
head = head.Successor()
}
-
return first, nil
}
@@ -182,6 +193,7 @@ func (cds *compositeDashboardSchema) LatestPanelSchemaFor(id string) (schema.Ver
}
latest := schema.Find(psch, schema.Latest())
+ // FIXME this relies on old sloppiness
sch := &genericVersionedSchema{
actual: cds.base.CUE().LookupPath(panelSubpath).Unify(mapPanelModel(id, latest)),
}
diff --git a/pkg/schema/load/load_test.go b/pkg/schema/load/load_test.go
index 12a01600f02..bf1be2d8f49 100644
--- a/pkg/schema/load/load_test.go
+++ b/pkg/schema/load/load_test.go
@@ -50,8 +50,6 @@ func TestScuemataBasics(t *testing.T) {
}
func TestDevenvDashboardValidity(t *testing.T) {
- t.Skip()
-
validdir := filepath.Join("..", "..", "..", "devenv", "dev-dashboards")
doTest := func(sch schema.VersionedCueSchema) func(t *testing.T) {
@@ -109,11 +107,9 @@ func TestDevenvDashboardValidity(t *testing.T) {
// TODO will need to expand this appropriately when the scuemata contain
// more than one schema
- // TODO disabled because base variant validation currently must fail in order for
- // dist/instance validation to do closed validation of plugin-specified fields
- // t.Run("base", doTest(dash))
- // dash, err := BaseDashboardFamily(p)
- // require.NoError(t, err, "error while loading base dashboard scuemata")
+ dash, err := BaseDashboardFamily(p)
+ require.NoError(t, err, "error while loading base dashboard scuemata")
+ t.Run("base", doTest(dash))
ddash, err := DistDashboardFamily(p)
require.NoError(t, err, "error while loading dist dashboard scuemata")
diff --git a/pkg/schema/load/panel.go b/pkg/schema/load/panel.go
index 8d6755fa407..0a176cd33e7 100644
--- a/pkg/schema/load/panel.go
+++ b/pkg/schema/load/panel.go
@@ -13,34 +13,10 @@ import (
"github.com/grafana/grafana/pkg/schema"
)
-// Returns a disjunction of structs representing each panel schema version
-// (post-mapping from on-disk #PanelModel form) from each scuemata in the map.
-func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Value, error) {
- partsi, err := rt.Compile("glue-panelDisjunction", `
- allPanels: [Name=_]: {}
- parts: or([for v in allPanels { v }])
- `)
- if err != nil {
- return cue.Value{}, err
- }
-
- parts := partsi.Value()
- for id, sch := range scuemap {
- for sch != nil {
- cv := mapPanelModel(id, sch)
-
- mjv, miv := sch.Version()
- parts = parts.FillPath(cue.MakePath(cue.Str("allPanels"), cue.Str(fmt.Sprintf("%s@%v.%v", id, mjv, miv))), cv)
- sch = sch.Successor()
- }
- }
-
- return parts.LookupPath(cue.MakePath(cue.Str("parts"))), nil
-}
-
// mapPanelModel maps a schema from the #PanelModel form in which it's declared
// in a plugin's model.cue to the structure in which it actually appears in the
// dashboard schema.
+// TODO remove, this is old sloppy hacks
func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
maj, min := vcs.Version()
// Ignore err return, this can't fail to compile
@@ -69,7 +45,7 @@ func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
return inter.Value().FillPath(cue.MakePath(cue.Str("in"), cue.Str("model")), vcs.CUE()).LookupPath(cue.MakePath(cue.Str(("result"))))
}
-func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, error) {
+func loadPanelScuemata(p BaseLoadPaths) (map[string]cue.Value, error) {
overlay := make(map[string]load.Source)
if err := toOverlay(prefix, p.BaseCueFS, overlay); err != nil {
@@ -89,7 +65,7 @@ func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, err
return nil, errors.New("could not locate #PanelFamily definition")
}
- all := make(map[string]schema.VersionedCueSchema)
+ all := make(map[string]cue.Value)
err = fs.WalkDir(p.DistPluginCueFS, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
@@ -138,7 +114,7 @@ func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, err
}
// Get the Family declaration in the models.cue file...
- pmod := imod.Value().LookupPath(cue.MakePath(cue.Str("Family")))
+ pmod := imod.Value().LookupPath(cue.MakePath(cue.Str("Panel")))
if !pmod.Exists() {
return fmt.Errorf("%s does not contain a declaration of its models at path 'Family'", path)
}
@@ -149,13 +125,8 @@ func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, err
return err
}
- // Create a generic schema family to represent the whole of the
- fam, err := buildGenericScuemata(pmod)
- if err != nil {
- return err
- }
+ all[id] = pmod
- all[id] = fam
return nil
})
if err != nil {
diff --git a/pkg/server/backgroundsvcs/background_services.go b/pkg/server/backgroundsvcs/background_services.go
index b8d67de3e7f..7d2e3440b6d 100644
--- a/pkg/server/backgroundsvcs/background_services.go
+++ b/pkg/server/backgroundsvcs/background_services.go
@@ -23,10 +23,14 @@ import (
"github.com/grafana/grafana/pkg/tsdb/azuremonitor"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
"github.com/grafana/grafana/pkg/tsdb/elasticsearch"
+ "github.com/grafana/grafana/pkg/tsdb/grafanads"
"github.com/grafana/grafana/pkg/tsdb/graphite"
"github.com/grafana/grafana/pkg/tsdb/influxdb"
"github.com/grafana/grafana/pkg/tsdb/loki"
+ "github.com/grafana/grafana/pkg/tsdb/mssql"
+ "github.com/grafana/grafana/pkg/tsdb/mysql"
"github.com/grafana/grafana/pkg/tsdb/opentsdb"
+ "github.com/grafana/grafana/pkg/tsdb/postgres"
"github.com/grafana/grafana/pkg/tsdb/prometheus"
"github.com/grafana/grafana/pkg/tsdb/tempo"
"github.com/grafana/grafana/pkg/tsdb/testdatasource"
@@ -43,6 +47,7 @@ func ProvideBackgroundServiceRegistry(
_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,
_ *testdatasource.TestDataPlugin, _ *plugindashboards.Service, _ *dashboardsnapshots.Service,
+ _ *postgres.Service, _ *mysql.Service, _ *mssql.Service, _ *grafanads.Service,
) *BackgroundServiceRegistry {
return NewBackgroundServiceRegistry(
diff --git a/pkg/server/wire.go b/pkg/server/wire.go
index 7ec5adc8ea5..ceb7a104966 100644
--- a/pkg/server/wire.go
+++ b/pkg/server/wire.go
@@ -57,9 +57,12 @@ import (
"github.com/grafana/grafana/pkg/tsdb/cloudmonitoring"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch"
"github.com/grafana/grafana/pkg/tsdb/elasticsearch"
+ "github.com/grafana/grafana/pkg/tsdb/grafanads"
"github.com/grafana/grafana/pkg/tsdb/graphite"
"github.com/grafana/grafana/pkg/tsdb/influxdb"
"github.com/grafana/grafana/pkg/tsdb/loki"
+ "github.com/grafana/grafana/pkg/tsdb/mssql"
+ "github.com/grafana/grafana/pkg/tsdb/mysql"
"github.com/grafana/grafana/pkg/tsdb/opentsdb"
"github.com/grafana/grafana/pkg/tsdb/postgres"
"github.com/grafana/grafana/pkg/tsdb/prometheus"
@@ -95,6 +98,8 @@ var wireBasicSet = wire.NewSet(
cloudmonitoring.ProvideService,
azuremonitor.ProvideService,
postgres.ProvideService,
+ mysql.ProvideService,
+ mssql.ProvideService,
httpclientprovider.New,
wire.Bind(new(httpclient.Provider), new(*sdkhttpclient.Provider)),
serverlock.ProvideService,
@@ -137,6 +142,7 @@ var wireBasicSet = wire.NewSet(
graphite.ProvideService,
prometheus.ProvideService,
elasticsearch.ProvideService,
+ grafanads.ProvideService,
dashboardsnapshots.ProvideService,
)
diff --git a/pkg/services/accesscontrol/mock/mock.go b/pkg/services/accesscontrol/mock/mock.go
index e2f6d91c0e8..1cf6ebcf856 100644
--- a/pkg/services/accesscontrol/mock/mock.go
+++ b/pkg/services/accesscontrol/mock/mock.go
@@ -42,8 +42,6 @@ type Mock struct {
RegisterFixedRolesFunc func() error
}
-type MockOptions func(*Mock)
-
// Ensure the mock stays in line with the interface
var _ fullAccessControl = New()
diff --git a/pkg/services/alerting/conditions/query.go b/pkg/services/alerting/conditions/query.go
index e7efbae16fb..8d37c3a9ea3 100644
--- a/pkg/services/alerting/conditions/query.go
+++ b/pkg/services/alerting/conditions/query.go
@@ -110,12 +110,17 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler plug
}
func calculateInterval(timeRange plugins.DataTimeRange, model *simplejson.Json, dsInfo *models.DataSource) (time.Duration, error) {
+ // if there is no min-interval specified in the datasource or in the dashboard-panel,
+ // the value of 1ms is used (this is how it is done in the dashboard-interval-calculation too,
+ // see https://github.com/grafana/grafana/blob/9a0040c0aeaae8357c650cec2ee644a571dddf3d/packages/grafana-data/src/datetime/rangeutil.ts#L264)
+ defaultMinInterval := time.Millisecond * 1
+
// interval.GetIntervalFrom has two problems (but they do not affect us here):
// - it returns the min-interval, so it should be called interval.GetMinIntervalFrom
// - it falls back to model.intervalMs. it should not, because that one is the real final
// interval-value calculated by the browser. but, in this specific case (old-alert),
// that value is not set, so the fallback never happens.
- minInterval, err := interval.GetIntervalFrom(dsInfo, model, time.Duration(0))
+ minInterval, err := interval.GetIntervalFrom(dsInfo, model, defaultMinInterval)
if err != nil {
return time.Duration(0), err
@@ -123,10 +128,7 @@ func calculateInterval(timeRange plugins.DataTimeRange, model *simplejson.Json,
calc := interval.NewCalculator()
- interval, err := calc.Calculate(timeRange, minInterval, "min")
- if err != nil {
- return time.Duration(0), err
- }
+ interval := calc.Calculate(timeRange, minInterval)
return interval.Value, nil
}
@@ -267,7 +269,8 @@ func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, t
},
},
Headers: map[string]string{
- "FromAlert": "true",
+ "FromAlert": "true",
+ "X-Cache-Skip": "true",
},
Debug: debug,
}
diff --git a/pkg/services/alerting/conditions/query_interval_test.go b/pkg/services/alerting/conditions/query_interval_test.go
index 72b9d3e76dd..7736da437e3 100644
--- a/pkg/services/alerting/conditions/query_interval_test.go
+++ b/pkg/services/alerting/conditions/query_interval_test.go
@@ -15,12 +15,6 @@ import (
. "github.com/smartystreets/goconvey/convey"
)
-// the time-range is 5m (300seconds) for every test-case,
-// maxDataPoints is 1500 for every test-case,
-// so the interval for this simple case should be 300s/1500 = 200ms,
-// but in some cases this is overridden by dashboard-panel-min-interval
-// or by datasource-min-interval
-
func TestQueryInterval(t *testing.T) {
Convey("When evaluating query condition, regarding the interval value", t, func() {
Convey("Can handle interval-calculation with no panel-min-interval and no datasource-min-interval", func() {
@@ -30,6 +24,8 @@ func TestQueryInterval(t *testing.T) {
// no datasource-min-interval
var dataSourceJson *simplejson.Json = nil
+ timeRange := "5m"
+
verifier := func(query plugins.DataSubQuery) {
// 5minutes timerange = 300000milliseconds; default-resolution is 1500pixels,
// so we should have 300000/1500 = 200milliseconds here
@@ -37,7 +33,7 @@ func TestQueryInterval(t *testing.T) {
So(query.MaxDataPoints, ShouldEqual, interval.DefaultRes)
}
- applyScenario(dataSourceJson, queryModel, verifier)
+ applyScenario(timeRange, dataSourceJson, queryModel, verifier)
})
Convey("Can handle interval-calculation with panel-min-interval and no datasource-min-interval", func() {
// panel-min-interval in the queryModel
@@ -46,12 +42,14 @@ func TestQueryInterval(t *testing.T) {
// no datasource-min-interval
var dataSourceJson *simplejson.Json = nil
+ timeRange := "5m"
+
verifier := func(query plugins.DataSubQuery) {
So(query.IntervalMS, ShouldEqual, 123000)
So(query.MaxDataPoints, ShouldEqual, interval.DefaultRes)
}
- applyScenario(dataSourceJson, queryModel, verifier)
+ applyScenario(timeRange, dataSourceJson, queryModel, verifier)
})
Convey("Can handle interval-calculation with no panel-min-interval and datasource-min-interval", func() {
// no panel-min-interval in the queryModel
@@ -63,12 +61,14 @@ func TestQueryInterval(t *testing.T) {
}`))
So(err, ShouldBeNil)
+ timeRange := "5m"
+
verifier := func(query plugins.DataSubQuery) {
So(query.IntervalMS, ShouldEqual, 71000)
So(query.MaxDataPoints, ShouldEqual, interval.DefaultRes)
}
- applyScenario(dataSourceJson, queryModel, verifier)
+ applyScenario(timeRange, dataSourceJson, queryModel, verifier)
})
Convey("Can handle interval-calculation with both panel-min-interval and datasource-min-interval", func() {
// panel-min-interval in the queryModel
@@ -80,6 +80,8 @@ func TestQueryInterval(t *testing.T) {
}`))
So(err, ShouldBeNil)
+ timeRange := "5m"
+
verifier := func(query plugins.DataSubQuery) {
// when both panel-min-interval and datasource-min-interval exists,
// panel-min-interval is used
@@ -87,7 +89,26 @@ func TestQueryInterval(t *testing.T) {
So(query.MaxDataPoints, ShouldEqual, interval.DefaultRes)
}
- applyScenario(dataSourceJson, queryModel, verifier)
+ applyScenario(timeRange, dataSourceJson, queryModel, verifier)
+ })
+
+ Convey("Can handle no min-interval, and very small time-ranges, where the default-min-interval=1ms applies", func() {
+ // no panel-min-interval in the queryModel
+ queryModel := `{"target": "aliasByNode(statsd.fakesite.counters.session_start.mobile.count, 4)"}`
+
+ // no datasource-min-interval
+ var dataSourceJson *simplejson.Json = nil
+
+ timeRange := "1s"
+
+ verifier := func(query plugins.DataSubQuery) {
+ // no min-interval exists, the default-min-interval will be used,
+ // and for such a short time-range this will cause the value to be 1millisecond.
+ So(query.IntervalMS, ShouldEqual, 1)
+ So(query.MaxDataPoints, ShouldEqual, interval.DefaultRes)
+ }
+
+ applyScenario(timeRange, dataSourceJson, queryModel, verifier)
})
})
}
@@ -114,7 +135,7 @@ func (rh fakeIntervalTestReqHandler) HandleRequest(ctx context.Context, dsInfo *
}
//nolint: staticcheck // plugins.DataResponse deprecated
-func applyScenario(dataSourceJsonData *simplejson.Json, queryModel string, verifier func(query plugins.DataSubQuery)) {
+func applyScenario(timeRange string, dataSourceJsonData *simplejson.Json, queryModel string, verifier func(query plugins.DataSubQuery)) {
Convey("desc", func() {
bus.AddHandler("test", func(query *models.GetDataSourceQuery) error {
query.Result = &models.DataSource{Id: 1, Type: "graphite", JsonData: dataSourceJsonData}
@@ -130,7 +151,7 @@ func applyScenario(dataSourceJsonData *simplejson.Json, queryModel string, verif
jsonModel, err := simplejson.NewJson([]byte(`{
"type": "query",
"query": {
- "params": ["A", "5m", "now"],
+ "params": ["A", "` + timeRange + `", "now"],
"datasourceId": 1,
"model": ` + queryModel + `
},
diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go
index efc4957377f..ade5c888590 100644
--- a/pkg/services/cleanup/cleanup.go
+++ b/pkg/services/cleanup/cleanup.go
@@ -2,6 +2,7 @@ package cleanup
import (
"context"
+ "errors"
"io/ioutil"
"os"
"path"
@@ -67,7 +68,7 @@ func (srv *CleanUpService) Run(ctx context.Context) error {
func (srv *CleanUpService) cleanUpOldAnnotations(ctx context.Context) {
cleaner := annotations.GetAnnotationCleaner()
affected, affectedTags, err := cleaner.CleanAnnotations(ctx, srv.Cfg)
- if err != nil {
+ if err != nil && !errors.Is(err, context.DeadlineExceeded) {
srv.log.Error("failed to clean up old annotations", "error", err)
} else {
srv.log.Debug("Deleted excess annotations", "annotations affected", affected, "annotation tags affected", affectedTags)
diff --git a/pkg/services/contexthandler/auth_proxy_test.go b/pkg/services/contexthandler/auth_proxy_test.go
index 072a0004283..1d4b7ce82b2 100644
--- a/pkg/services/contexthandler/auth_proxy_test.go
+++ b/pkg/services/contexthandler/auth_proxy_test.go
@@ -58,11 +58,8 @@ func TestInitContextWithAuthProxy_CachedInvalidUserID(t *testing.T) {
req, err := http.NewRequest("POST", "http://example.com", nil)
require.NoError(t, err)
ctx := &models.ReqContext{
- Context: &macaron.Context{
- Req: req,
- Data: map[string]interface{}{},
- },
- Logger: log.New("Test"),
+ Context: &macaron.Context{Req: req},
+ Logger: log.New("Test"),
}
req.Header.Set(svc.Cfg.AuthProxyHeaderName, name)
h, err := authproxy.HashCacheKey(name)
diff --git a/pkg/services/contexthandler/contexthandler.go b/pkg/services/contexthandler/contexthandler.go
index a593d673f2f..f8253df52cf 100644
--- a/pkg/services/contexthandler/contexthandler.go
+++ b/pkg/services/contexthandler/contexthandler.go
@@ -121,7 +121,6 @@ func (h *ContextHandler) Middleware(mContext *macaron.Context) {
}
reqContext.Logger = log.New("context", "userId", reqContext.UserId, "orgId", reqContext.OrgId, "uname", reqContext.Login)
- reqContext.Data["ctx"] = reqContext
span.LogFields(
ol.String("uname", reqContext.Login),
@@ -299,7 +298,7 @@ func (h *ContextHandler) initContextWithToken(reqContext *models.ReqContext, org
token, err := h.AuthTokenService.LookupToken(ctx, rawToken)
if err != nil {
reqContext.Logger.Error("Failed to look up user based on cookie", "error", err)
- reqContext.Data["lookupTokenErr"] = err
+ reqContext.LookupTokenErr = err
return false
}
diff --git a/pkg/services/dashboards/dashboard_service.go b/pkg/services/dashboards/dashboard_service.go
index d3e3a7aa3d2..2049082f667 100644
--- a/pkg/services/dashboards/dashboard_service.go
+++ b/pkg/services/dashboards/dashboard_service.go
@@ -103,7 +103,7 @@ func (dr *dashboardServiceImpl) buildSaveDashboardCommand(dto *SaveDashboardDTO,
if !util.IsValidShortUID(dash.Uid) {
return nil, models.ErrDashboardInvalidUid
- } else if len(dash.Uid) > 40 {
+ } else if util.IsShortUIDTooLong(dash.Uid) {
return nil, models.ErrDashboardUidTooLong
}
diff --git a/pkg/services/ldap/ldap.go b/pkg/services/ldap/ldap.go
index 6caf5494205..1ad3fa15905 100644
--- a/pkg/services/ldap/ldap.go
+++ b/pkg/services/ldap/ldap.go
@@ -12,9 +12,10 @@ import (
"strings"
"github.com/davecgh/go-spew/spew"
+ "gopkg.in/ldap.v3"
+
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
- "gopkg.in/ldap.v3"
)
// IConnection is interface for LDAP connection manipulation
@@ -252,16 +253,11 @@ func (server *Server) Users(logins []string) (
[]*models.ExternalUserInfo,
error,
) {
- var users []*ldap.Entry
+ var users [][]*ldap.Entry
err := getUsersIteration(logins, func(previous, current int) error {
- entries, err := server.users(logins[previous:current])
- if err != nil {
- return err
- }
-
- users = append(users, entries...)
-
- return nil
+ var err error
+ users, err = server.users(logins[previous:current])
+ return err
})
if err != nil {
return nil, err
@@ -308,13 +304,15 @@ func getUsersIteration(logins []string, fn func(int, int) error) error {
// users is helper method for the Users()
func (server *Server) users(logins []string) (
- []*ldap.Entry,
+ [][]*ldap.Entry,
error,
) {
var result *ldap.SearchResult
var Config = server.Config
var err error
+ var entries = make([][]*ldap.Entry, 0, len(Config.SearchBaseDNs))
+
for _, base := range Config.SearchBaseDNs {
result, err = server.Connection.Search(
server.getSearchRequest(base, logins),
@@ -324,11 +322,11 @@ func (server *Server) users(logins []string) (
}
if len(result.Entries) > 0 {
- break
+ entries = append(entries, result.Entries)
}
}
- return result.Entries, nil
+ return entries, nil
}
// validateGrafanaUser validates user access.
@@ -557,17 +555,26 @@ func (server *Server) requestMemberOf(entry *ldap.Entry) ([]string, error) {
// serializeUsers serializes the users
// from LDAP result to ExternalInfo struct
func (server *Server) serializeUsers(
- entries []*ldap.Entry,
+ entries [][]*ldap.Entry,
) ([]*models.ExternalUserInfo, error) {
var serialized []*models.ExternalUserInfo
+ var users = map[string]struct{}{}
- for _, user := range entries {
- extUser, err := server.buildGrafanaUser(user)
- if err != nil {
- return nil, err
+ for _, dn := range entries {
+ for _, user := range dn {
+ extUser, err := server.buildGrafanaUser(user)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, exists := users[extUser.Login]; exists {
+ // ignore duplicates
+ continue
+ }
+ users[extUser.Login] = struct{}{}
+
+ serialized = append(serialized, extUser)
}
-
- serialized = append(serialized, extUser)
}
return serialized, nil
diff --git a/pkg/services/ldap/ldap_helpers_test.go b/pkg/services/ldap/ldap_helpers_test.go
index 5062623d546..e276917c973 100644
--- a/pkg/services/ldap/ldap_helpers_test.go
+++ b/pkg/services/ldap/ldap_helpers_test.go
@@ -1,191 +1,141 @@
package ldap
import (
+ "fmt"
"testing"
- . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
"gopkg.in/ldap.v3"
)
-func TestLDAPHelpers(t *testing.T) {
- Convey("isMemberOf()", t, func() {
- Convey("Wildcard", func() {
- result := isMemberOf([]string{}, "*")
- So(result, ShouldBeTrue)
- })
+func TestIsMemberOf(t *testing.T) {
+ tests := []struct {
+ memberOf []string
+ group string
+ expected bool
+ }{
+ {memberOf: []string{}, group: "*", expected: true},
+ {memberOf: []string{"one", "Two", "three"}, group: "two", expected: true},
+ {memberOf: []string{"one", "Two", "three"}, group: "twos", expected: false},
+ }
- Convey("Should find one", func() {
- result := isMemberOf([]string{"one", "Two", "three"}, "two")
- So(result, ShouldBeTrue)
+ for _, tc := range tests {
+ t.Run(fmt.Sprintf("isMemberOf(%v, \"%s\") = %v", tc.memberOf, tc.group, tc.expected), func(t *testing.T) {
+ assert.Equal(t, tc.expected, isMemberOf(tc.memberOf, tc.group))
})
+ }
+}
- Convey("Should not find one", func() {
- result := isMemberOf([]string{"one", "Two", "three"}, "twos")
- So(result, ShouldBeFalse)
- })
- })
+func TestGetUsersIteration(t *testing.T) {
+ const pageSize = UsersMaxRequest
+ iterations := map[int]int{
+ 0: 0,
+ 400: 1,
+ 600: 2,
+ 1500: 3,
+ }
+
+ for userCount, expectedIterations := range iterations {
+ t.Run(fmt.Sprintf("getUserIteration iterates %d times for %d users", expectedIterations, userCount), func(t *testing.T) {
+ logins := make([]string, userCount)
- Convey("getUsersIteration()", t, func() {
- Convey("it should execute twice for 600 users", func() {
- logins := make([]string, 600)
i := 0
+ _ = getUsersIteration(logins, func(first int, last int) error {
+ assert.Equal(t, pageSize*i, first)
- result := getUsersIteration(logins, func(previous, current int) error {
- i++
-
- if i == 1 {
- So(previous, ShouldEqual, 0)
- So(current, ShouldEqual, 500)
- } else {
- So(previous, ShouldEqual, 500)
- So(current, ShouldEqual, 600)
+ expectedLast := pageSize*i + pageSize
+ if expectedLast > userCount {
+ expectedLast = userCount
}
- return nil
- })
+ assert.Equal(t, expectedLast, last)
- So(i, ShouldEqual, 2)
- So(result, ShouldBeNil)
- })
-
- Convey("it should execute three times for 1500 users", func() {
- logins := make([]string, 1500)
- i := 0
-
- result := getUsersIteration(logins, func(previous, current int) error {
- i++
- switch i {
- case 1:
- So(previous, ShouldEqual, 0)
- So(current, ShouldEqual, 500)
- case 2:
- So(previous, ShouldEqual, 500)
- So(current, ShouldEqual, 1000)
- default:
- So(previous, ShouldEqual, 1000)
- So(current, ShouldEqual, 1500)
- }
-
- return nil
- })
-
- So(i, ShouldEqual, 3)
- So(result, ShouldBeNil)
- })
-
- Convey("it should execute once for 400 users", func() {
- logins := make([]string, 400)
- i := 0
-
- result := getUsersIteration(logins, func(previous, current int) error {
- i++
- if i == 1 {
- So(previous, ShouldEqual, 0)
- So(current, ShouldEqual, 400)
- }
-
- return nil
- })
-
- So(i, ShouldEqual, 1)
- So(result, ShouldBeNil)
- })
-
- Convey("it should not execute for 0 users", func() {
- logins := make([]string, 0)
- i := 0
-
- result := getUsersIteration(logins, func(previous, current int) error {
i++
return nil
})
- So(i, ShouldEqual, 0)
- So(result, ShouldBeNil)
+ assert.Equal(t, expectedIterations, i)
})
+ }
+}
+
+func TestGetAttribute(t *testing.T) {
+ t.Run("DN", func(t *testing.T) {
+ entry := &ldap.Entry{
+ DN: "test",
+ }
+
+ result := getAttribute("dn", entry)
+ assert.Equal(t, "test", result)
})
- Convey("getAttribute()", t, func() {
- Convey("Should get DN", func() {
- entry := &ldap.Entry{
- DN: "test",
- }
-
- result := getAttribute("dn", entry)
-
- So(result, ShouldEqual, "test")
- })
-
- Convey("Should get username", func() {
- value := []string{"roelgerrits"}
- entry := &ldap.Entry{
- Attributes: []*ldap.EntryAttribute{
- {
- Name: "username", Values: value,
- },
+ t.Run("username", func(t *testing.T) {
+ value := "roelgerrits"
+ entry := &ldap.Entry{
+ Attributes: []*ldap.EntryAttribute{
+ {
+ Name: "username", Values: []string{value},
},
- }
+ },
+ }
- result := getAttribute("username", entry)
-
- So(result, ShouldEqual, value[0])
- })
-
- Convey("Should not get anything", func() {
- value := []string{"roelgerrits"}
- entry := &ldap.Entry{
- Attributes: []*ldap.EntryAttribute{
- {
- Name: "killa", Values: value,
- },
- },
- }
-
- result := getAttribute("username", entry)
-
- So(result, ShouldEqual, "")
- })
+ result := getAttribute("username", entry)
+ assert.Equal(t, value, result)
})
- Convey("getArrayAttribute()", t, func() {
- Convey("Should get DN", func() {
- entry := &ldap.Entry{
- DN: "test",
- }
-
- result := getArrayAttribute("dn", entry)
-
- So(result, ShouldResemble, []string{"test"})
- })
-
- Convey("Should get username", func() {
- value := []string{"roelgerrits"}
- entry := &ldap.Entry{
- Attributes: []*ldap.EntryAttribute{
- {
- Name: "username", Values: value,
- },
+ t.Run("no result", func(t *testing.T) {
+ value := []string{"roelgerrits"}
+ entry := &ldap.Entry{
+ Attributes: []*ldap.EntryAttribute{
+ {
+ Name: "killa", Values: value,
},
- }
+ },
+ }
- result := getArrayAttribute("username", entry)
-
- So(result, ShouldResemble, value)
- })
-
- Convey("Should not get anything", func() {
- value := []string{"roelgerrits"}
- entry := &ldap.Entry{
- Attributes: []*ldap.EntryAttribute{
- {
- Name: "username", Values: value,
- },
+ result := getAttribute("username", entry)
+ assert.Empty(t, result)
+ })
+}
+
+func TestGetArrayAttribute(t *testing.T) {
+ t.Run("DN", func(t *testing.T) {
+ entry := &ldap.Entry{
+ DN: "test",
+ }
+
+ result := getArrayAttribute("dn", entry)
+
+ assert.EqualValues(t, []string{"test"}, result)
+ })
+
+ t.Run("username", func(t *testing.T) {
+ value := []string{"roelgerrits"}
+ entry := &ldap.Entry{
+ Attributes: []*ldap.EntryAttribute{
+ {
+ Name: "username", Values: value,
+ },
+ },
+ }
+
+ result := getArrayAttribute("username", entry)
+
+ assert.EqualValues(t, value, result)
+ })
+
+ t.Run("no result", func(t *testing.T) {
+ value := []string{"roelgerrits"}
+ entry := &ldap.Entry{
+ Attributes: []*ldap.EntryAttribute{
+ {
+ Name: "username", Values: value,
},
- }
+ },
+ }
- result := getArrayAttribute("something", entry)
+ result := getArrayAttribute("something", entry)
- So(result, ShouldResemble, []string{})
- })
+ assert.Empty(t, result)
})
}
diff --git a/pkg/services/ldap/ldap_login_test.go b/pkg/services/ldap/ldap_login_test.go
index dea64fab48c..7b552a8edfa 100644
--- a/pkg/services/ldap/ldap_login_test.go
+++ b/pkg/services/ldap/ldap_login_test.go
@@ -4,231 +4,227 @@ import (
"errors"
"testing"
- . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
"gopkg.in/ldap.v3"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
)
-func TestLDAPLogin(t *testing.T) {
- defaultLogin := &models.LoginUserQuery{
- Username: "user",
- Password: "pwd",
- IpAddress: "192.168.1.1:56433",
+var defaultLogin = &models.LoginUserQuery{
+ Username: "user",
+ Password: "pwd",
+ IpAddress: "192.168.1.1:56433",
+}
+
+func TestServer_Login_UserBind_Fail(t *testing.T) {
+ connection := &MockConnection{}
+ entry := ldap.Entry{}
+ result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
+ connection.setSearchResult(&result)
+
+ connection.BindProvider = func(username, password string) error {
+ return &ldap.Error{
+ ResultCode: 49,
+ }
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
}
- Convey("Login()", t, func() {
- Convey("Should get invalid credentials when userBind fails", func() {
- connection := &MockConnection{}
- entry := ldap.Entry{}
- result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
- connection.setSearchResult(&result)
+ _, err := server.Login(defaultLogin)
- connection.BindProvider = func(username, password string) error {
- return &ldap.Error{
- ResultCode: 49,
- }
- }
- server := &Server{
- Config: &ServerConfig{
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- _, err := server.Login(defaultLogin)
-
- So(err, ShouldEqual, ErrInvalidCredentials)
- })
-
- Convey("Returns an error when search didn't find anything", func() {
- connection := &MockConnection{}
- result := ldap.SearchResult{Entries: []*ldap.Entry{}}
- connection.setSearchResult(&result)
-
- connection.BindProvider = func(username, password string) error {
- return nil
- }
- server := &Server{
- Config: &ServerConfig{
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- _, err := server.Login(defaultLogin)
-
- So(err, ShouldEqual, ErrCouldNotFindUser)
- })
-
- Convey("When search returns an error", func() {
- connection := &MockConnection{}
- expected := errors.New("Killa-gorilla")
- connection.setSearchError(expected)
-
- connection.BindProvider = func(username, password string) error {
- return nil
- }
- server := &Server{
- Config: &ServerConfig{
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- _, err := server.Login(defaultLogin)
-
- So(err, ShouldEqual, expected)
- })
-
- Convey("When login with valid credentials", func() {
- connection := &MockConnection{}
- entry := ldap.Entry{
- DN: "dn", Attributes: []*ldap.EntryAttribute{
- {Name: "username", Values: []string{"markelog"}},
- {Name: "surname", Values: []string{"Gaidarenko"}},
- {Name: "email", Values: []string{"markelog@gmail.com"}},
- {Name: "name", Values: []string{"Oleg"}},
- {Name: "memberof", Values: []string{"admins"}},
- },
- }
- result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
- connection.setSearchResult(&result)
-
- connection.BindProvider = func(username, password string) error {
- return nil
- }
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{
- Username: "username",
- Name: "name",
- MemberOf: "memberof",
- },
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- resp, err := server.Login(defaultLogin)
-
- So(err, ShouldBeNil)
- So(resp.Login, ShouldEqual, "markelog")
- })
-
- Convey("Should perform unauthenticated bind without admin", func() {
- connection := &MockConnection{}
- entry := ldap.Entry{
- DN: "test",
- }
- result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
- connection.setSearchResult(&result)
-
- connection.UnauthenticatedBindProvider = func() error {
- return nil
- }
- server := &Server{
- Config: &ServerConfig{
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- user, err := server.Login(defaultLogin)
-
- So(err, ShouldBeNil)
- So(user.AuthId, ShouldEqual, "test")
- So(connection.UnauthenticatedBindCalled, ShouldBeTrue)
- })
-
- Convey("Should perform authenticated binds", func() {
- connection := &MockConnection{}
- entry := ldap.Entry{
- DN: "test",
- }
- result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
- connection.setSearchResult(&result)
-
- adminUsername := ""
- adminPassword := ""
- username := ""
- password := ""
-
- i := 0
- connection.BindProvider = func(name, pass string) error {
- i++
- if i == 1 {
- adminUsername = name
- adminPassword = pass
- }
-
- if i == 2 {
- username = name
- password = pass
- }
-
- return nil
- }
- server := &Server{
- Config: &ServerConfig{
- BindDN: "killa",
- BindPassword: "gorilla",
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- user, err := server.Login(defaultLogin)
-
- So(err, ShouldBeNil)
-
- So(user.AuthId, ShouldEqual, "test")
- So(connection.BindCalled, ShouldBeTrue)
-
- So(adminUsername, ShouldEqual, "killa")
- So(adminPassword, ShouldEqual, "gorilla")
-
- So(username, ShouldEqual, "test")
- So(password, ShouldEqual, "pwd")
- })
- Convey("Should bind with user if %s exists in the bind_dn", func() {
- connection := &MockConnection{}
- entry := ldap.Entry{
- DN: "test",
- }
- connection.setSearchResult(&ldap.SearchResult{Entries: []*ldap.Entry{&entry}})
-
- authBindUser := ""
- authBindPassword := ""
-
- connection.BindProvider = func(name, pass string) error {
- authBindUser = name
- authBindPassword = pass
- return nil
- }
- server := &Server{
- Config: &ServerConfig{
- BindDN: "cn=%s,ou=users,dc=grafana,dc=org",
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: connection,
- log: log.New("test-logger"),
- }
-
- _, err := server.Login(defaultLogin)
-
- So(err, ShouldBeNil)
-
- So(authBindUser, ShouldEqual, "cn=user,ou=users,dc=grafana,dc=org")
- So(authBindPassword, ShouldEqual, "pwd")
- So(connection.BindCalled, ShouldBeTrue)
- })
- })
+ assert.ErrorIs(t, err, ErrInvalidCredentials)
+}
+
+func TestServer_Login_Search_NoResult(t *testing.T) {
+ connection := &MockConnection{}
+ result := ldap.SearchResult{Entries: []*ldap.Entry{}}
+ connection.setSearchResult(&result)
+
+ connection.BindProvider = func(username, password string) error {
+ return nil
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
+ }
+
+ _, err := server.Login(defaultLogin)
+ assert.ErrorIs(t, err, ErrCouldNotFindUser)
+}
+
+func TestServer_Login_Search_Error(t *testing.T) {
+ connection := &MockConnection{}
+ expected := errors.New("Killa-gorilla")
+ connection.setSearchError(expected)
+
+ connection.BindProvider = func(username, password string) error {
+ return nil
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
+ }
+
+ _, err := server.Login(defaultLogin)
+ assert.ErrorIs(t, err, expected)
+}
+
+func TestServer_Login_ValidCredentials(t *testing.T) {
+ connection := &MockConnection{}
+ entry := ldap.Entry{
+ DN: "dn", Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"markelog"}},
+ {Name: "surname", Values: []string{"Gaidarenko"}},
+ {Name: "email", Values: []string{"markelog@gmail.com"}},
+ {Name: "name", Values: []string{"Oleg"}},
+ {Name: "memberof", Values: []string{"admins"}},
+ },
+ }
+ result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
+ connection.setSearchResult(&result)
+
+ connection.BindProvider = func(username, password string) error {
+ return nil
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
+ MemberOf: "memberof",
+ },
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
+ }
+
+ resp, err := server.Login(defaultLogin)
+ require.NoError(t, err)
+ assert.Equal(t, "markelog", resp.Login)
+}
+
+// TestServer_Login_UnauthenticatedBind tests that unauthenticated bind
+// is called when there is no admin password or user wildcard in the
+// bind_dn.
+func TestServer_Login_UnauthenticatedBind(t *testing.T) {
+ connection := &MockConnection{}
+ entry := ldap.Entry{
+ DN: "test",
+ }
+ result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
+ connection.setSearchResult(&result)
+
+ connection.UnauthenticatedBindProvider = func() error {
+ return nil
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
+ }
+
+ user, err := server.Login(defaultLogin)
+ require.NoError(t, err)
+ assert.Equal(t, "test", user.AuthId)
+ assert.True(t, connection.UnauthenticatedBindCalled)
+}
+
+func TestServer_Login_AuthenticatedBind(t *testing.T) {
+ connection := &MockConnection{}
+ entry := ldap.Entry{
+ DN: "test",
+ }
+ result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
+ connection.setSearchResult(&result)
+
+ adminUsername := ""
+ adminPassword := ""
+ username := ""
+ password := ""
+
+ i := 0
+ connection.BindProvider = func(name, pass string) error {
+ i++
+ if i == 1 {
+ adminUsername = name
+ adminPassword = pass
+ }
+
+ if i == 2 {
+ username = name
+ password = pass
+ }
+
+ return nil
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ BindDN: "killa",
+ BindPassword: "gorilla",
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
+ }
+
+ user, err := server.Login(defaultLogin)
+ require.NoError(t, err)
+
+ assert.Equal(t, "test", user.AuthId)
+ assert.True(t, connection.BindCalled)
+
+ assert.Equal(t, "killa", adminUsername)
+ assert.Equal(t, "gorilla", adminPassword)
+
+ assert.Equal(t, "test", username)
+ assert.Equal(t, "pwd", password)
+}
+
+func TestServer_Login_UserWildcardBind(t *testing.T) {
+ connection := &MockConnection{}
+ entry := ldap.Entry{
+ DN: "test",
+ }
+ connection.setSearchResult(&ldap.SearchResult{Entries: []*ldap.Entry{&entry}})
+
+ authBindUser := ""
+ authBindPassword := ""
+
+ connection.BindProvider = func(name, pass string) error {
+ authBindUser = name
+ authBindPassword = pass
+ return nil
+ }
+ server := &Server{
+ Config: &ServerConfig{
+ BindDN: "cn=%s,ou=users,dc=grafana,dc=org",
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: connection,
+ log: log.New("test-logger"),
+ }
+
+ _, err := server.Login(defaultLogin)
+ require.NoError(t, err)
+
+ assert.Equal(t, "cn=user,ou=users,dc=grafana,dc=org", authBindUser)
+ assert.Equal(t, "pwd", authBindPassword)
+ assert.True(t, connection.BindCalled)
}
diff --git a/pkg/services/ldap/ldap_private_test.go b/pkg/services/ldap/ldap_private_test.go
index 431f94f0d94..d4d0f1c238d 100644
--- a/pkg/services/ldap/ldap_private_test.go
+++ b/pkg/services/ldap/ldap_private_test.go
@@ -3,271 +3,252 @@ package ldap
import (
"testing"
+ "github.com/stretchr/testify/require"
+
+ "github.com/stretchr/testify/assert"
+
+ "gopkg.in/ldap.v3"
+
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
- . "github.com/smartystreets/goconvey/convey"
- "gopkg.in/ldap.v3"
)
-func TestLDAPPrivateMethods(t *testing.T) {
- Convey("getSearchRequest()", t, func() {
- Convey("with enabled GroupSearchFilterUserAttribute setting", func() {
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{
- Username: "username",
- Name: "name",
- MemberOf: "memberof",
- Email: "email",
- },
- GroupSearchFilterUserAttribute: "gansta",
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- log: log.New("test-logger"),
- }
+func TestServer_getSearchRequest(t *testing.T) {
+ expected := &ldap.SearchRequest{
+ BaseDN: "killa",
+ Scope: 2,
+ DerefAliases: 0,
+ SizeLimit: 0,
+ TimeLimit: 0,
+ TypesOnly: false,
+ Filter: "(|)",
+ Attributes: []string{
+ "username",
+ "email",
+ "name",
+ "memberof",
+ "gansta",
+ },
+ Controls: nil,
+ }
- result := server.getSearchRequest("killa", []string{"gorilla"})
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
+ MemberOf: "memberof",
+ Email: "email",
+ },
+ GroupSearchFilterUserAttribute: "gansta",
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ log: log.New("test-logger"),
+ }
- So(result, ShouldResemble, &ldap.SearchRequest{
- BaseDN: "killa",
- Scope: 2,
- DerefAliases: 0,
- SizeLimit: 0,
- TimeLimit: 0,
- TypesOnly: false,
- Filter: "(|)",
- Attributes: []string{
- "username",
- "email",
- "name",
- "memberof",
- "gansta",
+ result := server.getSearchRequest("killa", []string{"gorilla"})
+
+ assert.EqualValues(t, expected, result)
+}
+
+func TestSerializeUsers(t *testing.T) {
+ t.Run("simple case", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
+ MemberOf: "memberof",
+ Email: "email",
},
- Controls: nil,
- })
- })
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: &MockConnection{},
+ log: log.New("test-logger"),
+ }
+
+ entry := ldap.Entry{
+ DN: "dn",
+ Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"roelgerrits"}},
+ {Name: "surname", Values: []string{"Gerrits"}},
+ {Name: "email", Values: []string{"roel@test.com"}},
+ {Name: "name", Values: []string{"Roel"}},
+ {Name: "memberof", Values: []string{"admins"}},
+ },
+ }
+ users := [][]*ldap.Entry{{&entry}}
+
+ result, err := server.serializeUsers(users)
+ require.NoError(t, err)
+
+ assert.Equal(t, "roelgerrits", result[0].Login)
+ assert.Equal(t, "roel@test.com", result[0].Email)
+ assert.Contains(t, result[0].Groups, "admins")
})
- Convey("serializeUsers()", t, func() {
- Convey("simple case", func() {
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{
- Username: "username",
- Name: "name",
- MemberOf: "memberof",
- Email: "email",
- },
- SearchBaseDNs: []string{"BaseDNHere"},
+ t.Run("without lastname", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
+ MemberOf: "memberof",
+ Email: "email",
},
- Connection: &MockConnection{},
- log: log.New("test-logger"),
- }
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: &MockConnection{},
+ log: log.New("test-logger"),
+ }
- entry := ldap.Entry{
- DN: "dn",
- Attributes: []*ldap.EntryAttribute{
- {Name: "username", Values: []string{"roelgerrits"}},
- {Name: "surname", Values: []string{"Gerrits"}},
- {Name: "email", Values: []string{"roel@test.com"}},
- {Name: "name", Values: []string{"Roel"}},
- {Name: "memberof", Values: []string{"admins"}},
- },
- }
- users := []*ldap.Entry{&entry}
+ entry := ldap.Entry{
+ DN: "dn",
+ Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"roelgerrits"}},
+ {Name: "email", Values: []string{"roel@test.com"}},
+ {Name: "name", Values: []string{"Roel"}},
+ {Name: "memberof", Values: []string{"admins"}},
+ },
+ }
+ users := [][]*ldap.Entry{{&entry}}
- result, err := server.serializeUsers(users)
+ result, err := server.serializeUsers(users)
+ require.NoError(t, err)
- So(err, ShouldBeNil)
- So(result[0].Login, ShouldEqual, "roelgerrits")
- So(result[0].Email, ShouldEqual, "roel@test.com")
- So(result[0].Groups, ShouldContain, "admins")
- })
-
- Convey("without lastname", func() {
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{
- Username: "username",
- Name: "name",
- MemberOf: "memberof",
- Email: "email",
- },
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: &MockConnection{},
- log: log.New("test-logger"),
- }
-
- entry := ldap.Entry{
- DN: "dn",
- Attributes: []*ldap.EntryAttribute{
- {Name: "username", Values: []string{"roelgerrits"}},
- {Name: "email", Values: []string{"roel@test.com"}},
- {Name: "name", Values: []string{"Roel"}},
- {Name: "memberof", Values: []string{"admins"}},
- },
- }
- users := []*ldap.Entry{&entry}
-
- result, err := server.serializeUsers(users)
-
- So(err, ShouldBeNil)
- So(result[0].IsDisabled, ShouldBeFalse)
- So(result[0].Name, ShouldEqual, "Roel")
- })
-
- Convey("a user without matching groups should be marked as disabled", func() {
- server := &Server{
- Config: &ServerConfig{
- Groups: []*GroupToOrgRole{{
- GroupDN: "foo",
- OrgId: 1,
- OrgRole: models.ROLE_EDITOR,
- }},
- },
- Connection: &MockConnection{},
- log: log.New("test-logger"),
- }
-
- entry := ldap.Entry{
- DN: "dn",
- Attributes: []*ldap.EntryAttribute{
- {Name: "memberof", Values: []string{"admins"}},
- },
- }
- users := []*ldap.Entry{&entry}
-
- result, err := server.serializeUsers(users)
-
- So(err, ShouldBeNil)
- So(len(result), ShouldEqual, 1)
- So(result[0].IsDisabled, ShouldBeTrue)
- })
+ assert.False(t, result[0].IsDisabled)
+ assert.Equal(t, "Roel", result[0].Name)
})
- Convey("validateGrafanaUser()", t, func() {
- Convey("Returns error when user does not belong in any of the specified LDAP groups", func() {
- server := &Server{
- Config: &ServerConfig{
- Groups: []*GroupToOrgRole{
- {
- OrgId: 1,
- },
- },
- },
- log: logger.New("test"),
- }
+ t.Run("mark user without matching group as disabled", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Groups: []*GroupToOrgRole{{
+ GroupDN: "foo",
+ OrgId: 1,
+ OrgRole: models.ROLE_EDITOR,
+ }},
+ },
+ Connection: &MockConnection{},
+ log: log.New("test-logger"),
+ }
- user := &models.ExternalUserInfo{
- Login: "markelog",
- }
+ entry := ldap.Entry{
+ DN: "dn",
+ Attributes: []*ldap.EntryAttribute{
+ {Name: "memberof", Values: []string{"admins"}},
+ },
+ }
+ users := [][]*ldap.Entry{{&entry}}
- result := server.validateGrafanaUser(user)
+ result, err := server.serializeUsers(users)
+ require.NoError(t, err)
- So(result, ShouldEqual, ErrInvalidCredentials)
- })
+ assert.Len(t, result, 1)
+ assert.True(t, result[0].IsDisabled)
+ })
+}
+
+func TestServer_validateGrafanaUser(t *testing.T) {
+ t.Run("no group config", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Groups: []*GroupToOrgRole{},
+ },
+ log: logger.New("test"),
+ }
+
+ user := &models.ExternalUserInfo{
+ Login: "markelog",
+ }
+
+ err := server.validateGrafanaUser(user)
+ require.NoError(t, err)
+ })
- Convey("Does not return error when group config is empty", func() {
- server := &Server{
- Config: &ServerConfig{
- Groups: []*GroupToOrgRole{},
+ t.Run("user in group", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Groups: []*GroupToOrgRole{
+ {
+ OrgId: 1,
+ },
},
- log: logger.New("test"),
- }
-
- user := &models.ExternalUserInfo{
- Login: "markelog",
- }
+ },
+ log: logger.New("test"),
+ }
+
+ user := &models.ExternalUserInfo{
+ Login: "markelog",
+ OrgRoles: map[int64]models.RoleType{
+ 1: "test",
+ },
+ }
+
+ err := server.validateGrafanaUser(user)
+ require.NoError(t, err)
+ })
- result := server.validateGrafanaUser(user)
-
- So(result, ShouldBeNil)
- })
-
- Convey("Does not return error when groups are there", func() {
- server := &Server{
- Config: &ServerConfig{
- Groups: []*GroupToOrgRole{
- {
- OrgId: 1,
- },
+ t.Run("user not in group", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Groups: []*GroupToOrgRole{
+ {
+ OrgId: 1,
},
},
- log: logger.New("test"),
- }
+ },
+ log: logger.New("test"),
+ }
- user := &models.ExternalUserInfo{
- Login: "markelog",
- OrgRoles: map[int64]models.RoleType{
- 1: "test",
- },
- }
+ user := &models.ExternalUserInfo{
+ Login: "markelog",
+ }
- result := server.validateGrafanaUser(user)
-
- So(result, ShouldBeNil)
- })
+ err := server.validateGrafanaUser(user)
+ require.ErrorIs(t, err, ErrInvalidCredentials)
+ })
+}
+
+func TestServer_binds(t *testing.T) {
+ t.Run("single bind with cn wildcard", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ BindDN: "cn=%s,dc=grafana,dc=org",
+ },
+ }
+
+ assert.True(t, server.shouldSingleBind())
+ assert.Equal(t, "cn=test,dc=grafana,dc=org", server.singleBindDN("test"))
})
- Convey("shouldAdminBind()", t, func() {
- Convey("it should require admin userBind", func() {
- server := &Server{
- Config: &ServerConfig{
- BindPassword: "test",
- },
- }
-
- result := server.shouldAdminBind()
- So(result, ShouldBeTrue)
- })
+ t.Run("don't single bind", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ BindDN: "cn=admin,dc=grafana,dc=org",
+ },
+ }
- Convey("it should not require admin userBind", func() {
- server := &Server{
- Config: &ServerConfig{
- BindPassword: "",
- },
- }
-
- result := server.shouldAdminBind()
- So(result, ShouldBeFalse)
- })
+ assert.False(t, server.shouldSingleBind())
})
- Convey("shouldSingleBind()", t, func() {
- Convey("it should allow single bind", func() {
- server := &Server{
- Config: &ServerConfig{
- BindDN: "cn=%s,dc=grafana,dc=org",
- },
- }
+ t.Run("admin user bind", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ BindPassword: "test",
+ },
+ }
- result := server.shouldSingleBind()
- So(result, ShouldBeTrue)
- })
-
- Convey("it should not allow single bind", func() {
- server := &Server{
- Config: &ServerConfig{
- BindDN: "cn=admin,dc=grafana,dc=org",
- },
- }
-
- result := server.shouldSingleBind()
- So(result, ShouldBeFalse)
- })
+ assert.True(t, server.shouldAdminBind())
})
- Convey("singleBindDN()", t, func() {
- Convey("it should allow single bind", func() {
- server := &Server{
- Config: &ServerConfig{
- BindDN: "cn=%s,dc=grafana,dc=org",
- },
- }
+ t.Run("don't admin user bind", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ BindPassword: "",
+ },
+ }
- result := server.singleBindDN("test")
- So(result, ShouldEqual, "cn=test,dc=grafana,dc=org")
- })
+ assert.False(t, server.shouldAdminBind())
})
}
diff --git a/pkg/services/ldap/ldap_test.go b/pkg/services/ldap/ldap_test.go
index ea1fd049bf3..042ac045506 100644
--- a/pkg/services/ldap/ldap_test.go
+++ b/pkg/services/ldap/ldap_test.go
@@ -2,226 +2,319 @@ package ldap
import (
"errors"
+ "fmt"
"testing"
- "github.com/grafana/grafana/pkg/infra/log"
- . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"gopkg.in/ldap.v3"
+
+ "github.com/grafana/grafana/pkg/infra/log"
)
-func TestPublicAPI(t *testing.T) {
- Convey("New()", t, func() {
- Convey("Should return ", func() {
- result := New(&ServerConfig{
+func TestNew(t *testing.T) {
+ result := New(&ServerConfig{
+ Attr: AttributeMap{},
+ SearchBaseDNs: []string{"BaseDNHere"},
+ })
+
+ assert.Implements(t, (*IServer)(nil), result)
+}
+
+func TestServer_Close(t *testing.T) {
+ t.Run("close the connection", func(t *testing.T) {
+ connection := &MockConnection{}
+
+ server := &Server{
+ Config: &ServerConfig{
Attr: AttributeMap{},
SearchBaseDNs: []string{"BaseDNHere"},
- })
+ },
+ Connection: connection,
+ }
- So(result, ShouldImplement, (*IServer)(nil))
- })
+ assert.NotPanics(t, server.Close)
+ assert.True(t, connection.CloseCalled)
})
- Convey("Close()", t, func() {
- Convey("Should close the connection", func() {
- connection := &MockConnection{}
+ t.Run("panic if no connection", func(t *testing.T) {
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{},
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: nil,
+ }
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{},
- SearchBaseDNs: []string{"BaseDNHere"},
+ assert.Panics(t, server.Close)
+ })
+}
+
+func TestServer_Users(t *testing.T) {
+ t.Run("one user", func(t *testing.T) {
+ conn := &MockConnection{}
+ entry := ldap.Entry{
+ DN: "dn", Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"roelgerrits"}},
+ {Name: "surname", Values: []string{"Gerrits"}},
+ {Name: "email", Values: []string{"roel@test.com"}},
+ {Name: "name", Values: []string{"Roel"}},
+ {Name: "memberof", Values: []string{"admins"}},
+ }}
+ result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
+ conn.setSearchResult(&result)
+
+ // Set up attribute map without surname and email
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
+ MemberOf: "memberof",
},
- Connection: connection,
- }
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: conn,
+ log: log.New("test-logger"),
+ }
- So(server.Close, ShouldNotPanic)
- So(connection.CloseCalled, ShouldBeTrue)
- })
+ searchResult, err := server.Users([]string{"roelgerrits"})
- Convey("Should panic if no connection is established", func() {
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{},
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: nil,
- }
+ require.NoError(t, err)
+ assert.NotNil(t, searchResult)
- So(server.Close, ShouldPanic)
- })
+ // User should be searched in ldap
+ assert.True(t, conn.SearchCalled)
+ // No empty attributes should be added to the search request
+ assert.Len(t, conn.SearchAttributes, 3)
})
- Convey("Users()", t, func() {
- Convey("Finds one user", func() {
- MockConnection := &MockConnection{}
- entry := ldap.Entry{
- DN: "dn", Attributes: []*ldap.EntryAttribute{
- {Name: "username", Values: []string{"roelgerrits"}},
- {Name: "surname", Values: []string{"Gerrits"}},
- {Name: "email", Values: []string{"roel@test.com"}},
- {Name: "name", Values: []string{"Roel"}},
- {Name: "memberof", Values: []string{"admins"}},
- }}
- result := ldap.SearchResult{Entries: []*ldap.Entry{&entry}}
- MockConnection.setSearchResult(&result)
-
- // Set up attribute map without surname and email
- server := &Server{
- Config: &ServerConfig{
- Attr: AttributeMap{
- Username: "username",
- Name: "name",
- MemberOf: "memberof",
- },
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: MockConnection,
- log: log.New("test-logger"),
- }
+
+ t.Run("error", func(t *testing.T) {
+ expected := errors.New("Killa-gorilla")
+ conn := &MockConnection{}
+ conn.setSearchError(expected)
+
+ // Set up attribute map without surname and email
+ server := &Server{
+ Config: &ServerConfig{
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: conn,
+ log: log.New("test-logger"),
+ }
+
+ _, err := server.Users([]string{"roelgerrits"})
- searchResult, err := server.Users([]string{"roelgerrits"})
+ assert.ErrorIs(t, err, expected)
+ })
- So(err, ShouldBeNil)
- So(searchResult, ShouldNotBeNil)
+ t.Run("no user", func(t *testing.T) {
+ conn := &MockConnection{}
+ result := ldap.SearchResult{Entries: []*ldap.Entry{}}
+ conn.setSearchResult(&result)
- // User should be searched in ldap
- So(MockConnection.SearchCalled, ShouldBeTrue)
+ // Set up attribute map without surname and email
+ server := &Server{
+ Config: &ServerConfig{
+ SearchBaseDNs: []string{"BaseDNHere"},
+ },
+ Connection: conn,
+ log: log.New("test-logger"),
+ }
- // No empty attributes should be added to the search request
- So(len(MockConnection.SearchAttributes), ShouldEqual, 3)
- })
+ searchResult, err := server.Users([]string{"roelgerrits"})
- Convey("Handles a error", func() {
- expected := errors.New("Killa-gorilla")
- MockConnection := &MockConnection{}
- MockConnection.setSearchError(expected)
+ require.NoError(t, err)
+ assert.Empty(t, searchResult)
+ })
- // Set up attribute map without surname and email
- server := &Server{
- Config: &ServerConfig{
- SearchBaseDNs: []string{"BaseDNHere"},
- },
- Connection: MockConnection,
- log: log.New("test-logger"),
- }
+ t.Run("multiple DNs", func(t *testing.T) {
+ conn := &MockConnection{}
+ serviceDN := "dc=svc,dc=example,dc=org"
+ serviceEntry := ldap.Entry{
+ DN: "dn", Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"imgrenderer"}},
+ {Name: "name", Values: []string{"Image renderer"}},
+ }}
+ services := ldap.SearchResult{Entries: []*ldap.Entry{&serviceEntry}}
- _, err := server.Users([]string{"roelgerrits"})
+ userDN := "dc=users,dc=example,dc=org"
+ userEntry := ldap.Entry{
+ DN: "dn", Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"grot"}},
+ {Name: "name", Values: []string{"Grot"}},
+ }}
+ users := ldap.SearchResult{Entries: []*ldap.Entry{&userEntry}}
- So(err, ShouldEqual, expected)
+ conn.setSearchFunc(func(request *ldap.SearchRequest) (*ldap.SearchResult, error) {
+ switch request.BaseDN {
+ case userDN:
+ return &users, nil
+ case serviceDN:
+ return &services, nil
+ default:
+ return nil, fmt.Errorf("test case not defined for baseDN: '%s'", request.BaseDN)
+ }
})
-
- Convey("Should return empty slice if none were found", func() {
- MockConnection := &MockConnection{}
- result := ldap.SearchResult{Entries: []*ldap.Entry{}}
- MockConnection.setSearchResult(&result)
- // Set up attribute map without surname and email
- server := &Server{
- Config: &ServerConfig{
- SearchBaseDNs: []string{"BaseDNHere"},
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
},
- Connection: MockConnection,
- log: log.New("test-logger"),
- }
+ SearchBaseDNs: []string{serviceDN, userDN},
+ },
+ Connection: conn,
+ log: log.New("test-logger"),
+ }
- searchResult, err := server.Users([]string{"roelgerrits"})
+ searchResult, err := server.Users([]string{"imgrenderer", "grot"})
+ require.NoError(t, err)
- So(err, ShouldBeNil)
- So(searchResult, ShouldBeEmpty)
- })
+ assert.Len(t, searchResult, 2)
})
+
+ t.Run("same user in multiple DNs", func(t *testing.T) {
+ conn := &MockConnection{}
+ firstDN := "dc=users1,dc=example,dc=org"
+ firstEntry := ldap.Entry{
+ DN: "dn", Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"grot"}},
+ {Name: "name", Values: []string{"Grot the First"}},
+ }}
+ firsts := ldap.SearchResult{Entries: []*ldap.Entry{&firstEntry}}
+
+ secondDN := "dc=users2,dc=example,dc=org"
+ secondEntry := ldap.Entry{
+ DN: "dn", Attributes: []*ldap.EntryAttribute{
+ {Name: "username", Values: []string{"grot"}},
+ {Name: "name", Values: []string{"Grot the Second"}},
+ }}
+ seconds := ldap.SearchResult{Entries: []*ldap.Entry{&secondEntry}}
- Convey("UserBind()", t, func() {
- Convey("Should use provided DN and password", func() {
- connection := &MockConnection{}
- var actualUsername, actualPassword string
- connection.BindProvider = func(username, password string) error {
- actualUsername = username
- actualPassword = password
- return nil
+ conn.setSearchFunc(func(request *ldap.SearchRequest) (*ldap.SearchResult, error) {
+ switch request.BaseDN {
+ case secondDN:
+ return &seconds, nil
+ case firstDN:
+ return &firsts, nil
+ default:
+ return nil, fmt.Errorf("test case not defined for baseDN: '%s'", request.BaseDN)
}
- server := &Server{
- Connection: connection,
- Config: &ServerConfig{
- BindDN: "cn=admin,dc=grafana,dc=org",
+ })
+
+ server := &Server{
+ Config: &ServerConfig{
+ Attr: AttributeMap{
+ Username: "username",
+ Name: "name",
},
- }
+ SearchBaseDNs: []string{firstDN, secondDN},
+ },
+ Connection: conn,
+ log: log.New("test-logger"),
+ }
+
+ res, err := server.Users([]string{"grot"})
+ require.NoError(t, err)
+ require.Len(t, res, 1)
+ assert.Equal(t, "Grot the First", res[0].Name)
+ })
+}
+
+func TestServer_UserBind(t *testing.T) {
+ t.Run("use provided DN and password", func(t *testing.T) {
+ connection := &MockConnection{}
+ var actualUsername, actualPassword string
+ connection.BindProvider = func(username, password string) error {
+ actualUsername = username
+ actualPassword = password
+ return nil
+ }
+ server := &Server{
+ Connection: connection,
+ Config: &ServerConfig{
+ BindDN: "cn=admin,dc=grafana,dc=org",
+ },
+ }
- dn := "cn=user,ou=users,dc=grafana,dc=org"
- err := server.UserBind(dn, "pwd")
+ dn := "cn=user,ou=users,dc=grafana,dc=org"
+ err := server.UserBind(dn, "pwd")
- So(err, ShouldBeNil)
- So(actualUsername, ShouldEqual, dn)
- So(actualPassword, ShouldEqual, "pwd")
- })
+ require.NoError(t, err)
+ assert.Equal(t, dn, actualUsername)
+ assert.Equal(t, "pwd", actualPassword)
+ })
- Convey("Should handle an error", func() {
- connection := &MockConnection{}
- expected := &ldap.Error{
- ResultCode: uint16(25),
- }
- connection.BindProvider = func(username, password string) error {
- return expected
- }
- server := &Server{
- Connection: connection,
- Config: &ServerConfig{
- BindDN: "cn=%s,ou=users,dc=grafana,dc=org",
- },
- log: log.New("test-logger"),
- }
- err := server.UserBind("user", "pwd")
- So(err, ShouldEqual, expected)
- })
+ t.Run("error", func(t *testing.T) {
+ connection := &MockConnection{}
+ expected := &ldap.Error{
+ ResultCode: uint16(25),
+ }
+ connection.BindProvider = func(username, password string) error {
+ return expected
+ }
+ server := &Server{
+ Connection: connection,
+ Config: &ServerConfig{
+ BindDN: "cn=%s,ou=users,dc=grafana,dc=org",
+ },
+ log: log.New("test-logger"),
+ }
+ err := server.UserBind("user", "pwd")
+ assert.ErrorIs(t, err, expected)
})
+}
- Convey("AdminBind()", t, func() {
- Convey("Should use admin DN and password", func() {
- connection := &MockConnection{}
- var actualUsername, actualPassword string
- connection.BindProvider = func(username, password string) error {
- actualUsername = username
- actualPassword = password
- return nil
- }
+func TestServer_AdminBind(t *testing.T) {
+ t.Run("use admin DN and password", func(t *testing.T) {
+ connection := &MockConnection{}
+ var actualUsername, actualPassword string
+ connection.BindProvider = func(username, password string) error {
+ actualUsername = username
+ actualPassword = password
+ return nil
+ }
- dn := "cn=admin,dc=grafana,dc=org"
+ dn := "cn=admin,dc=grafana,dc=org"
- server := &Server{
- Connection: connection,
- Config: &ServerConfig{
- BindPassword: "pwd",
- BindDN: dn,
- },
- }
+ server := &Server{
+ Connection: connection,
+ Config: &ServerConfig{
+ BindPassword: "pwd",
+ BindDN: dn,
+ },
+ }
- err := server.AdminBind()
+ err := server.AdminBind()
+ require.NoError(t, err)
- So(err, ShouldBeNil)
- So(actualUsername, ShouldEqual, dn)
- So(actualPassword, ShouldEqual, "pwd")
- })
+ assert.Equal(t, dn, actualUsername)
+ assert.Equal(t, "pwd", actualPassword)
+ })
- Convey("Should handle an error", func() {
- connection := &MockConnection{}
- expected := &ldap.Error{
- ResultCode: uint16(25),
- }
- connection.BindProvider = func(username, password string) error {
- return expected
- }
+ t.Run("error", func(t *testing.T) {
+ connection := &MockConnection{}
+ expected := &ldap.Error{
+ ResultCode: uint16(25),
+ }
+ connection.BindProvider = func(username, password string) error {
+ return expected
+ }
- dn := "cn=admin,dc=grafana,dc=org"
+ dn := "cn=admin,dc=grafana,dc=org"
- server := &Server{
- Connection: connection,
- Config: &ServerConfig{
- BindPassword: "pwd",
- BindDN: dn,
- },
- log: log.New("test-logger"),
- }
+ server := &Server{
+ Connection: connection,
+ Config: &ServerConfig{
+ BindPassword: "pwd",
+ BindDN: dn,
+ },
+ log: log.New("test-logger"),
+ }
- err := server.AdminBind()
- So(err, ShouldEqual, expected)
- })
+ err := server.AdminBind()
+ assert.ErrorIs(t, err, expected)
})
}
diff --git a/pkg/services/ldap/testing.go b/pkg/services/ldap/testing.go
index 8bad83a2d92..cd9ff9184f4 100644
--- a/pkg/services/ldap/testing.go
+++ b/pkg/services/ldap/testing.go
@@ -6,10 +6,11 @@ import (
"gopkg.in/ldap.v3"
)
+type searchFunc = func(request *ldap.SearchRequest) (*ldap.SearchResult, error)
+
// MockConnection struct for testing
type MockConnection struct {
- SearchResult *ldap.SearchResult
- SearchError error
+ SearchFunc searchFunc
SearchCalled bool
SearchAttributes []string
@@ -56,11 +57,19 @@ func (c *MockConnection) Close() {
}
func (c *MockConnection) setSearchResult(result *ldap.SearchResult) {
- c.SearchResult = result
+ c.SearchFunc = func(request *ldap.SearchRequest) (*ldap.SearchResult, error) {
+ return result, nil
+ }
}
func (c *MockConnection) setSearchError(err error) {
- c.SearchError = err
+ c.SearchFunc = func(request *ldap.SearchRequest) (*ldap.SearchResult, error) {
+ return nil, err
+ }
+}
+
+func (c *MockConnection) setSearchFunc(fn searchFunc) {
+ c.SearchFunc = fn
}
// Search mocks Search connection function
@@ -68,11 +77,7 @@ func (c *MockConnection) Search(sr *ldap.SearchRequest) (*ldap.SearchResult, err
c.SearchCalled = true
c.SearchAttributes = sr.Attributes
- if c.SearchError != nil {
- return nil, c.SearchError
- }
-
- return c.SearchResult, nil
+ return c.SearchFunc(sr)
}
// Add mocks Add connection function
diff --git a/pkg/services/libraryelements/api.go b/pkg/services/libraryelements/api.go
index 983ad3d6694..4df43e3760f 100644
--- a/pkg/services/libraryelements/api.go
+++ b/pkg/services/libraryelements/api.go
@@ -125,5 +125,11 @@ func toLibraryElementError(err error, message string) response.Response {
if errors.Is(err, errLibraryElementHasConnections) {
return response.Error(403, errLibraryElementHasConnections.Error(), err)
}
+ if errors.Is(err, errLibraryElementInvalidUID) {
+ return response.Error(400, errLibraryElementInvalidUID.Error(), err)
+ }
+ if errors.Is(err, errLibraryElementUIDTooLong) {
+ return response.Error(400, errLibraryElementUIDTooLong.Error(), err)
+ }
return response.Error(500, message, err)
}
diff --git a/pkg/services/libraryelements/database.go b/pkg/services/libraryelements/database.go
index 3fa51f77d85..ddc5ed18342 100644
--- a/pkg/services/libraryelements/database.go
+++ b/pkg/services/libraryelements/database.go
@@ -2,6 +2,7 @@ package libraryelements
import (
"encoding/json"
+ "errors"
"fmt"
"strings"
"time"
@@ -92,10 +93,20 @@ func (l *LibraryElementService) createLibraryElement(c *models.ReqContext, cmd C
if err := l.requireSupportedElementKind(cmd.Kind); err != nil {
return LibraryElementDTO{}, err
}
+ createUID := cmd.UID
+ if len(createUID) == 0 {
+ createUID = util.GenerateShortUID()
+ } else {
+ if !util.IsValidShortUID(createUID) {
+ return LibraryElementDTO{}, errLibraryElementInvalidUID
+ } else if util.IsShortUIDTooLong(createUID) {
+ return LibraryElementDTO{}, errLibraryElementUIDTooLong
+ }
+ }
element := LibraryElement{
OrgID: c.SignedInUser.OrgId,
FolderID: cmd.FolderID,
- UID: util.GenerateShortUID(),
+ UID: createUID,
Name: cmd.Name,
Model: cmd.Model,
Version: 1,
@@ -434,12 +445,27 @@ func (l *LibraryElementService) patchLibraryElement(c *models.ReqContext, cmd pa
if elementInDB.Version != cmd.Version {
return errLibraryElementVersionMismatch
}
+ updateUID := cmd.UID
+ if len(updateUID) == 0 {
+ updateUID = uid
+ } else if updateUID != uid {
+ if !util.IsValidShortUID(updateUID) {
+ return errLibraryElementInvalidUID
+ } else if util.IsShortUIDTooLong(updateUID) {
+ return errLibraryElementUIDTooLong
+ }
+
+ _, err := getLibraryElement(l.SQLStore.Dialect, session, updateUID, c.SignedInUser.OrgId)
+ if !errors.Is(err, errLibraryElementNotFound) {
+ return errLibraryElementAlreadyExists
+ }
+ }
var libraryElement = LibraryElement{
ID: elementInDB.ID,
OrgID: c.SignedInUser.OrgId,
FolderID: cmd.FolderID,
- UID: uid,
+ UID: updateUID,
Name: cmd.Name,
Kind: elementInDB.Kind,
Type: elementInDB.Type,
diff --git a/pkg/services/libraryelements/libraryelements_create_test.go b/pkg/services/libraryelements/libraryelements_create_test.go
index 080905e5b25..01f72f35963 100644
--- a/pkg/services/libraryelements/libraryelements_create_test.go
+++ b/pkg/services/libraryelements/libraryelements_create_test.go
@@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/util"
)
func TestCreateLibraryElement(t *testing.T) {
@@ -59,6 +60,76 @@ func TestCreateLibraryElement(t *testing.T) {
}
})
+ testScenario(t, "When an admin tries to create a library panel that does not exists using an nonexistent UID, it should succeed",
+ func(t *testing.T, sc scenarioContext) {
+ command := getCreatePanelCommand(sc.folder.Id, "Nonexistent UID")
+ command.UID = util.GenerateShortUID()
+ resp := sc.service.createHandler(sc.reqContext, command)
+ var result = validateAndUnMarshalResponse(t, resp)
+ var expected = libraryElementResult{
+ Result: libraryElement{
+ ID: 1,
+ OrgID: 1,
+ FolderID: 1,
+ UID: command.UID,
+ Name: "Nonexistent UID",
+ Kind: int64(models.PanelElement),
+ Type: "text",
+ Description: "A description",
+ Model: map[string]interface{}{
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "description": "A description",
+ "id": float64(1),
+ "title": "Text - Library Panel",
+ "type": "text",
+ },
+ Version: 1,
+ Meta: LibraryElementDTOMeta{
+ ConnectedDashboards: 0,
+ Created: result.Result.Meta.Created,
+ Updated: result.Result.Meta.Updated,
+ CreatedBy: LibraryElementDTOMetaUser{
+ ID: 1,
+ Name: "signed_in_user",
+ AvatarURL: "/avatar/37524e1eb8b3e32850b57db0a19af93b",
+ },
+ UpdatedBy: LibraryElementDTOMetaUser{
+ ID: 1,
+ Name: "signed_in_user",
+ AvatarURL: "/avatar/37524e1eb8b3e32850b57db0a19af93b",
+ },
+ },
+ },
+ }
+ if diff := cmp.Diff(expected, result, getCompareOptions()...); diff != "" {
+ t.Fatalf("Result mismatch (-want +got):\n%s", diff)
+ }
+ })
+
+ scenarioWithPanel(t, "When an admin tries to create a library panel that does not exists using an existent UID, it should fail",
+ func(t *testing.T, sc scenarioContext) {
+ command := getCreatePanelCommand(sc.folder.Id, "Existing UID")
+ command.UID = sc.initialResult.Result.UID
+ resp := sc.service.createHandler(sc.reqContext, command)
+ require.Equal(t, 400, resp.Status())
+ })
+
+ scenarioWithPanel(t, "When an admin tries to create a library panel that does not exists using an invalid UID, it should fail",
+ func(t *testing.T, sc scenarioContext) {
+ command := getCreatePanelCommand(sc.folder.Id, "Invalid UID")
+ command.UID = "Testing an invalid UID"
+ resp := sc.service.createHandler(sc.reqContext, command)
+ require.Equal(t, 400, resp.Status())
+ })
+
+ scenarioWithPanel(t, "When an admin tries to create a library panel that does not exists using an UID that is too long, it should fail",
+ func(t *testing.T, sc scenarioContext) {
+ command := getCreatePanelCommand(sc.folder.Id, "Invalid UID")
+ command.UID = "j6T00KRZzj6T00KRZzj6T00KRZzj6T00KRZzj6T00K"
+ resp := sc.service.createHandler(sc.reqContext, command)
+ require.Equal(t, 400, resp.Status())
+ })
+
testScenario(t, "When an admin tries to create a library panel where name and panel title differ, it should not update panel title",
func(t *testing.T, sc scenarioContext) {
command := getCreatePanelCommand(1, "Library Panel Name")
diff --git a/pkg/services/libraryelements/libraryelements_patch_test.go b/pkg/services/libraryelements/libraryelements_patch_test.go
index 4b2c3d58eda..b644e1caa23 100644
--- a/pkg/services/libraryelements/libraryelements_patch_test.go
+++ b/pkg/services/libraryelements/libraryelements_patch_test.go
@@ -3,6 +3,8 @@ package libraryelements
import (
"testing"
+ "github.com/grafana/grafana/pkg/util"
+
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
@@ -122,6 +124,70 @@ func TestPatchLibraryElement(t *testing.T) {
}
})
+ scenarioWithPanel(t, "When an admin tries to patch a library panel with a nonexistent UID, it should change UID successfully and return correct result",
+ func(t *testing.T, sc scenarioContext) {
+ cmd := patchLibraryElementCommand{
+ FolderID: -1,
+ UID: util.GenerateShortUID(),
+ Kind: int64(models.PanelElement),
+ Version: 1,
+ }
+ sc.reqContext.ReplaceAllParams(map[string]string{":uid": sc.initialResult.Result.UID})
+ resp := sc.service.patchHandler(sc.reqContext, cmd)
+ var result = validateAndUnMarshalResponse(t, resp)
+ sc.initialResult.Result.UID = cmd.UID
+ sc.initialResult.Result.Meta.CreatedBy.Name = userInDbName
+ sc.initialResult.Result.Meta.CreatedBy.AvatarURL = userInDbAvatar
+ sc.initialResult.Result.Model["title"] = "Text - Library Panel"
+ sc.initialResult.Result.Version = 2
+ if diff := cmp.Diff(sc.initialResult.Result, result.Result, getCompareOptions()...); diff != "" {
+ t.Fatalf("Result mismatch (-want +got):\n%s", diff)
+ }
+ })
+
+ scenarioWithPanel(t, "When an admin tries to patch a library panel with an invalid UID, it should fail",
+ func(t *testing.T, sc scenarioContext) {
+ cmd := patchLibraryElementCommand{
+ FolderID: -1,
+ UID: "Testing an invalid UID",
+ Kind: int64(models.PanelElement),
+ Version: 1,
+ }
+ sc.reqContext.ReplaceAllParams(map[string]string{":uid": sc.initialResult.Result.UID})
+ resp := sc.service.patchHandler(sc.reqContext, cmd)
+ require.Equal(t, 400, resp.Status())
+ })
+
+ scenarioWithPanel(t, "When an admin tries to patch a library panel with an UID that is too long, it should fail",
+ func(t *testing.T, sc scenarioContext) {
+ cmd := patchLibraryElementCommand{
+ FolderID: -1,
+ UID: "j6T00KRZzj6T00KRZzj6T00KRZzj6T00KRZzj6T00K",
+ Kind: int64(models.PanelElement),
+ Version: 1,
+ }
+ sc.reqContext.ReplaceAllParams(map[string]string{":uid": sc.initialResult.Result.UID})
+ resp := sc.service.patchHandler(sc.reqContext, cmd)
+ require.Equal(t, 400, resp.Status())
+ })
+
+ scenarioWithPanel(t, "When an admin tries to patch a library panel with an existing UID, it should fail",
+ func(t *testing.T, sc scenarioContext) {
+ command := getCreatePanelCommand(sc.folder.Id, "Existing UID")
+ command.UID = util.GenerateShortUID()
+ resp := sc.service.createHandler(sc.reqContext, command)
+ require.Equal(t, 200, resp.Status())
+ cmd := patchLibraryElementCommand{
+ FolderID: -1,
+ UID: command.UID,
+ Kind: int64(models.PanelElement),
+ Version: 1,
+ }
+ sc.reqContext.ReplaceAllParams(map[string]string{":uid": sc.initialResult.Result.UID})
+ resp = sc.service.patchHandler(sc.reqContext, cmd)
+ require.Equal(t, 400, resp.Status())
+ })
+
scenarioWithPanel(t, "When an admin tries to patch a library panel with model only, it should change model successfully, sync type and description fields and return correct result",
func(t *testing.T, sc scenarioContext) {
cmd := patchLibraryElementCommand{
diff --git a/pkg/services/libraryelements/models.go b/pkg/services/libraryelements/models.go
index 6cecb54991a..711e6616ec9 100644
--- a/pkg/services/libraryelements/models.go
+++ b/pkg/services/libraryelements/models.go
@@ -136,7 +136,7 @@ type LibraryElementConnectionDTO struct {
var (
// errLibraryElementAlreadyExists is an error for when the user tries to add a library element that already exists.
- errLibraryElementAlreadyExists = errors.New("library element with that name already exists")
+ errLibraryElementAlreadyExists = errors.New("library element with that name or UID already exists")
// errLibraryElementNotFound is an error for when a library element can't be found.
errLibraryElementNotFound = errors.New("library element could not be found")
// errLibraryElementDashboardNotFound is an error for when a library element connection can't be found.
@@ -147,8 +147,12 @@ var (
errLibraryElementVersionMismatch = errors.New("the library element has been changed by someone else")
// errLibraryElementUnSupportedElementKind is an error for when the kind is unsupported.
errLibraryElementUnSupportedElementKind = errors.New("the element kind is not supported")
- // ErrFolderHasConnectedLibraryElements is an error for when an user deletes a folder that contains connected library elements.
+ // ErrFolderHasConnectedLibraryElements is an error for when a user deletes a folder that contains connected library elements.
ErrFolderHasConnectedLibraryElements = errors.New("folder contains library elements that are linked in use")
+ // errLibraryElementInvalidUID is an error for when the uid of a library element is invalid
+ errLibraryElementInvalidUID = errors.New("uid contains illegal characters")
+ // errLibraryElementUIDTooLong is an error for when the uid of a library element is invalid
+ errLibraryElementUIDTooLong = errors.New("uid too long, max 40 characters")
)
// Commands
@@ -159,6 +163,7 @@ type CreateLibraryElementCommand struct {
Name string `json:"name"`
Model json.RawMessage `json:"model"`
Kind int64 `json:"kind" binding:"Required"`
+ UID string `json:"uid"`
}
// patchLibraryElementCommand is the command for patching a LibraryElement
@@ -168,6 +173,7 @@ type patchLibraryElementCommand struct {
Model json.RawMessage `json:"model"`
Kind int64 `json:"kind" binding:"Required"`
Version int64 `json:"version" binding:"Required"`
+ UID string `json:"uid"`
}
// searchLibraryElementsQuery is the query used for searching for Elements
diff --git a/pkg/services/librarypanels/librarypanels.go b/pkg/services/librarypanels/librarypanels.go
index 480f880b984..a8fc3c60897 100644
--- a/pkg/services/librarypanels/librarypanels.go
+++ b/pkg/services/librarypanels/librarypanels.go
@@ -47,11 +47,25 @@ func (lps *LibraryPanelService) LoadLibraryPanelsForDashboard(c *models.ReqConte
return err
}
- panels := dash.Data.Get("panels").MustArray()
+ return loadLibraryPanelsRecursively(elements, dash.Data)
+}
+
+func loadLibraryPanelsRecursively(elements map[string]libraryelements.LibraryElementDTO, parent *simplejson.Json) error {
+ panels := parent.Get("panels").MustArray()
for i, panel := range panels {
panelAsJSON := simplejson.NewFromAny(panel)
libraryPanel := panelAsJSON.Get("libraryPanel")
- if libraryPanel.Interface() == nil {
+ panelType := panelAsJSON.Get("type").MustString()
+ if !isLibraryPanelOrRow(libraryPanel, panelType) {
+ continue
+ }
+
+ // we have a row
+ if panelType == "row" {
+ err := loadLibraryPanelsRecursively(elements, panelAsJSON)
+ if err != nil {
+ return err
+ }
continue
}
@@ -64,7 +78,7 @@ func (lps *LibraryPanelService) LoadLibraryPanelsForDashboard(c *models.ReqConte
elementInDB, ok := elements[uid]
if !ok {
name := libraryPanel.Get("name").MustString()
- elem := dash.Data.Get("panels").GetIndex(i)
+ elem := parent.Get("panels").GetIndex(i)
elem.Set("gridPos", panelAsJSON.Get("gridPos").MustMap())
elem.Set("id", panelAsJSON.Get("id").MustInt64())
elem.Set("type", fmt.Sprintf("Name: \"%s\", UID: \"%s\"", name, uid))
@@ -91,10 +105,10 @@ func (lps *LibraryPanelService) LoadLibraryPanelsForDashboard(c *models.ReqConte
}
// set the library panel json as the new panel json in dashboard json
- dash.Data.Get("panels").SetIndex(i, libraryPanelModelAsJSON.Interface())
+ parent.Get("panels").SetIndex(i, libraryPanelModelAsJSON.Interface())
// set dashboard specific props
- elem := dash.Data.Get("panels").GetIndex(i)
+ elem := parent.Get("panels").GetIndex(i)
elem.Set("gridPos", panelAsJSON.Get("gridPos").MustMap())
elem.Set("id", panelAsJSON.Get("id").MustInt64())
elem.Set("libraryPanel", map[string]interface{}{
@@ -129,11 +143,25 @@ func (lps *LibraryPanelService) LoadLibraryPanelsForDashboard(c *models.ReqConte
// CleanLibraryPanelsForDashboard loops through all panels in dashboard JSON and cleans up any library panel JSON so that
// only the necessary JSON properties remain when storing the dashboard JSON.
func (lps *LibraryPanelService) CleanLibraryPanelsForDashboard(dash *models.Dashboard) error {
- panels := dash.Data.Get("panels").MustArray()
+ return cleanLibraryPanelsRecursively(dash.Data)
+}
+
+func cleanLibraryPanelsRecursively(parent *simplejson.Json) error {
+ panels := parent.Get("panels").MustArray()
for i, panel := range panels {
panelAsJSON := simplejson.NewFromAny(panel)
libraryPanel := panelAsJSON.Get("libraryPanel")
- if libraryPanel.Interface() == nil {
+ panelType := panelAsJSON.Get("type").MustString()
+ if !isLibraryPanelOrRow(libraryPanel, panelType) {
+ continue
+ }
+
+ // we have a row
+ if panelType == "row" {
+ err := cleanLibraryPanelsRecursively(panelAsJSON)
+ if err != nil {
+ return err
+ }
continue
}
@@ -150,7 +178,7 @@ func (lps *LibraryPanelService) CleanLibraryPanelsForDashboard(dash *models.Dash
// keep only the necessary JSON properties, the rest of the properties should be safely stored in library_panels table
gridPos := panelAsJSON.Get("gridPos").MustMap()
id := panelAsJSON.Get("id").MustInt64(int64(i))
- dash.Data.Get("panels").SetIndex(i, map[string]interface{}{
+ parent.Get("panels").SetIndex(i, map[string]interface{}{
"id": id,
"gridPos": gridPos,
"libraryPanel": map[string]interface{}{
@@ -167,10 +195,39 @@ func (lps *LibraryPanelService) CleanLibraryPanelsForDashboard(dash *models.Dash
func (lps *LibraryPanelService) ConnectLibraryPanelsForDashboard(c *models.ReqContext, dash *models.Dashboard) error {
panels := dash.Data.Get("panels").MustArray()
libraryPanels := make(map[string]string)
+ err := connectLibraryPanelsRecursively(c, panels, libraryPanels)
+ if err != nil {
+ return err
+ }
+
+ elementUIDs := make([]string, 0, len(libraryPanels))
+ for libraryPanel := range libraryPanels {
+ elementUIDs = append(elementUIDs, libraryPanel)
+ }
+
+ return lps.LibraryElementService.ConnectElementsToDashboard(c, elementUIDs, dash.Id)
+}
+
+func isLibraryPanelOrRow(panel *simplejson.Json, panelType string) bool {
+ return panel.Interface() != nil || panelType == "row"
+}
+
+func connectLibraryPanelsRecursively(c *models.ReqContext, panels []interface{}, libraryPanels map[string]string) error {
for _, panel := range panels {
panelAsJSON := simplejson.NewFromAny(panel)
libraryPanel := panelAsJSON.Get("libraryPanel")
- if libraryPanel.Interface() == nil {
+ panelType := panelAsJSON.Get("type").MustString()
+ if !isLibraryPanelOrRow(libraryPanel, panelType) {
+ continue
+ }
+
+ // we have a row
+ if panelType == "row" {
+ rowPanels := panelAsJSON.Get("panels").MustArray()
+ err := connectLibraryPanelsRecursively(c, rowPanels, libraryPanels)
+ if err != nil {
+ return err
+ }
continue
}
@@ -185,10 +242,5 @@ func (lps *LibraryPanelService) ConnectLibraryPanelsForDashboard(c *models.ReqCo
}
}
- elementUIDs := make([]string, 0, len(libraryPanels))
- for libraryPanel := range libraryPanels {
- elementUIDs = append(elementUIDs, libraryPanel)
- }
-
- return lps.LibraryElementService.ConnectElementsToDashboard(c, elementUIDs, dash.Id)
+ return nil
}
diff --git a/pkg/services/librarypanels/librarypanels_test.go b/pkg/services/librarypanels/librarypanels_test.go
index 467d5c15bb4..7d2e4f6e0b7 100644
--- a/pkg/services/librarypanels/librarypanels_test.go
+++ b/pkg/services/librarypanels/librarypanels_test.go
@@ -123,6 +123,219 @@ func TestLoadLibraryPanelsForDashboard(t *testing.T) {
}
})
+ scenarioWithLibraryPanel(t, "When an admin tries to load a dashboard with library panels inside and outside of rows, it should copy JSON properties from library panels",
+ func(t *testing.T, sc scenarioContext) {
+ cmd := libraryelements.CreateLibraryElementCommand{
+ FolderID: sc.initialResult.Result.FolderID,
+ Name: "Outside row",
+ Model: []byte(`
+ {
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "id": 1,
+ "title": "Text - Library Panel",
+ "type": "text",
+ "description": "A description"
+ }
+ `),
+ Kind: int64(models.PanelElement),
+ }
+ outsidePanel, err := sc.elementService.CreateElement(sc.reqContext, cmd)
+ require.NoError(t, err)
+ dashJSON := map[string]interface{}{
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(1),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 0,
+ },
+ },
+ map[string]interface{}{
+ "collapsed": true,
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 6,
+ },
+ "id": int64(2),
+ "type": "row",
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(3),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 7,
+ },
+ },
+ map[string]interface{}{
+ "id": int64(4),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 13,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "libraryPanel": map[string]interface{}{
+ "uid": sc.initialResult.Result.UID,
+ "name": sc.initialResult.Result.Name,
+ },
+ "title": "Inside row",
+ "type": "text",
+ },
+ },
+ },
+ map[string]interface{}{
+ "id": int64(5),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 19,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "libraryPanel": map[string]interface{}{
+ "uid": outsidePanel.UID,
+ "name": outsidePanel.Name,
+ },
+ "title": "Outside row",
+ "type": "text",
+ },
+ },
+ }
+ dash := models.Dashboard{
+ Title: "Testing LoadLibraryPanelsForDashboard",
+ Data: simplejson.NewFromAny(dashJSON),
+ }
+ dashInDB := createDashboard(t, sc.sqlStore, sc.user, &dash, sc.folder.Id)
+ err = sc.elementService.ConnectElementsToDashboard(sc.reqContext, []string{outsidePanel.UID, sc.initialResult.Result.UID}, dashInDB.Id)
+ require.NoError(t, err)
+
+ err = sc.service.LoadLibraryPanelsForDashboard(sc.reqContext, dashInDB)
+ require.NoError(t, err)
+ expectedJSON := map[string]interface{}{
+ "title": "Testing LoadLibraryPanelsForDashboard",
+ "uid": dashInDB.Uid,
+ "version": dashInDB.Version,
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(1),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 0,
+ },
+ },
+ map[string]interface{}{
+ "collapsed": true,
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 6,
+ },
+ "id": int64(2),
+ "type": "row",
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(3),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 7,
+ },
+ },
+ map[string]interface{}{
+ "id": int64(4),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 13,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "description": "A description",
+ "libraryPanel": map[string]interface{}{
+ "uid": sc.initialResult.Result.UID,
+ "name": sc.initialResult.Result.Name,
+ "type": sc.initialResult.Result.Type,
+ "description": sc.initialResult.Result.Description,
+ "version": sc.initialResult.Result.Version,
+ "meta": map[string]interface{}{
+ "folderName": "ScenarioFolder",
+ "folderUid": sc.folder.Uid,
+ "connectedDashboards": int64(1),
+ "created": sc.initialResult.Result.Meta.Created,
+ "updated": sc.initialResult.Result.Meta.Updated,
+ "createdBy": map[string]interface{}{
+ "id": sc.initialResult.Result.Meta.CreatedBy.ID,
+ "name": UserInDbName,
+ "avatarUrl": UserInDbAvatar,
+ },
+ "updatedBy": map[string]interface{}{
+ "id": sc.initialResult.Result.Meta.UpdatedBy.ID,
+ "name": UserInDbName,
+ "avatarUrl": UserInDbAvatar,
+ },
+ },
+ },
+ "title": "Text - Library Panel",
+ "type": "text",
+ },
+ },
+ },
+ map[string]interface{}{
+ "id": int64(5),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 19,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "description": "A description",
+ "libraryPanel": map[string]interface{}{
+ "uid": outsidePanel.UID,
+ "name": outsidePanel.Name,
+ "type": outsidePanel.Type,
+ "description": outsidePanel.Description,
+ "version": outsidePanel.Version,
+ "meta": map[string]interface{}{
+ "folderName": "ScenarioFolder",
+ "folderUid": sc.folder.Uid,
+ "connectedDashboards": int64(1),
+ "created": outsidePanel.Meta.Created,
+ "updated": outsidePanel.Meta.Updated,
+ "createdBy": map[string]interface{}{
+ "id": outsidePanel.Meta.CreatedBy.ID,
+ "name": UserInDbName,
+ "avatarUrl": UserInDbAvatar,
+ },
+ "updatedBy": map[string]interface{}{
+ "id": outsidePanel.Meta.UpdatedBy.ID,
+ "name": UserInDbName,
+ "avatarUrl": UserInDbAvatar,
+ },
+ },
+ },
+ "title": "Text - Library Panel",
+ "type": "text",
+ },
+ },
+ }
+ expected := simplejson.NewFromAny(expectedJSON)
+ if diff := cmp.Diff(expected.Interface(), dash.Data.Interface(), getCompareOptions()...); diff != "" {
+ t.Fatalf("Result mismatch (-want +got):\n%s", diff)
+ }
+ })
+
scenarioWithLibraryPanel(t, "When an admin tries to load a dashboard with a library panel without uid, it should fail",
func(t *testing.T, sc scenarioContext) {
dashJSON := map[string]interface{}{
@@ -310,6 +523,169 @@ func TestCleanLibraryPanelsForDashboard(t *testing.T) {
}
})
+ scenarioWithLibraryPanel(t, "When an admin tries to store a dashboard with library panels inside and outside of rows, it should just keep the correct JSON properties",
+ func(t *testing.T, sc scenarioContext) {
+ cmd := libraryelements.CreateLibraryElementCommand{
+ FolderID: sc.initialResult.Result.FolderID,
+ Name: "Outside row",
+ Model: []byte(`
+ {
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "id": 1,
+ "title": "Text - Library Panel",
+ "type": "text",
+ "description": "A description"
+ }
+ `),
+ Kind: int64(models.PanelElement),
+ }
+ outsidePanel, err := sc.elementService.CreateElement(sc.reqContext, cmd)
+ require.NoError(t, err)
+ dashJSON := map[string]interface{}{
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(1),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 0,
+ },
+ },
+ map[string]interface{}{
+ "collapsed": true,
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 6,
+ },
+ "id": int64(2),
+ "type": "row",
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(3),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 7,
+ },
+ },
+ map[string]interface{}{
+ "id": int64(4),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 13,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "libraryPanel": map[string]interface{}{
+ "uid": sc.initialResult.Result.UID,
+ "name": sc.initialResult.Result.Name,
+ },
+ "title": "Inside row",
+ "type": "text",
+ },
+ },
+ },
+ map[string]interface{}{
+ "id": int64(5),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 19,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "libraryPanel": map[string]interface{}{
+ "uid": outsidePanel.UID,
+ "name": outsidePanel.Name,
+ },
+ "title": "Outside row",
+ "type": "text",
+ },
+ },
+ }
+ dash := models.Dashboard{
+ Title: "Testing CleanLibraryPanelsForDashboard",
+ Data: simplejson.NewFromAny(dashJSON),
+ }
+ dashInDB := createDashboard(t, sc.sqlStore, sc.user, &dash, sc.folder.Id)
+
+ err = sc.service.CleanLibraryPanelsForDashboard(dashInDB)
+ require.NoError(t, err)
+ expectedJSON := map[string]interface{}{
+ "title": "Testing CleanLibraryPanelsForDashboard",
+ "uid": dashInDB.Uid,
+ "version": dashInDB.Version,
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(1),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 0,
+ },
+ },
+ map[string]interface{}{
+ "collapsed": true,
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 6,
+ },
+ "id": int64(2),
+ "type": "row",
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(3),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 7,
+ },
+ },
+ map[string]interface{}{
+ "id": int64(4),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 13,
+ },
+ "libraryPanel": map[string]interface{}{
+ "uid": sc.initialResult.Result.UID,
+ "name": sc.initialResult.Result.Name,
+ },
+ },
+ },
+ },
+ map[string]interface{}{
+ "id": int64(5),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 19,
+ },
+ "libraryPanel": map[string]interface{}{
+ "uid": outsidePanel.UID,
+ "name": outsidePanel.Name,
+ },
+ },
+ },
+ }
+ expected := simplejson.NewFromAny(expectedJSON)
+ if diff := cmp.Diff(expected.Interface(), dash.Data.Interface(), getCompareOptions()...); diff != "" {
+ t.Fatalf("Result mismatch (-want +got):\n%s", diff)
+ }
+ })
+
scenarioWithLibraryPanel(t, "When an admin tries to store a dashboard with a library panel without uid, it should fail",
func(t *testing.T, sc scenarioContext) {
dashJSON := map[string]interface{}{
@@ -438,6 +814,107 @@ func TestConnectLibraryPanelsForDashboard(t *testing.T) {
require.Equal(t, sc.initialResult.Result.UID, elements[sc.initialResult.Result.UID].UID)
})
+ scenarioWithLibraryPanel(t, "When an admin tries to store a dashboard with library panels inside and outside of rows, it should connect all",
+ func(t *testing.T, sc scenarioContext) {
+ cmd := libraryelements.CreateLibraryElementCommand{
+ FolderID: sc.initialResult.Result.FolderID,
+ Name: "Outside row",
+ Model: []byte(`
+ {
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "id": 1,
+ "title": "Text - Library Panel",
+ "type": "text",
+ "description": "A description"
+ }
+ `),
+ Kind: int64(models.PanelElement),
+ }
+ outsidePanel, err := sc.elementService.CreateElement(sc.reqContext, cmd)
+ require.NoError(t, err)
+ dashJSON := map[string]interface{}{
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(1),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 0,
+ },
+ },
+ map[string]interface{}{
+ "collapsed": true,
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 6,
+ },
+ "id": int64(2),
+ "type": "row",
+ "panels": []interface{}{
+ map[string]interface{}{
+ "id": int64(3),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 7,
+ },
+ },
+ map[string]interface{}{
+ "id": int64(4),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 13,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "libraryPanel": map[string]interface{}{
+ "uid": sc.initialResult.Result.UID,
+ "name": sc.initialResult.Result.Name,
+ },
+ "title": "Inside row",
+ "type": "text",
+ },
+ },
+ },
+ map[string]interface{}{
+ "id": int64(5),
+ "gridPos": map[string]interface{}{
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 19,
+ },
+ "datasource": "${DS_GDEV-TESTDATA}",
+ "libraryPanel": map[string]interface{}{
+ "uid": outsidePanel.UID,
+ "name": outsidePanel.Name,
+ },
+ "title": "Outside row",
+ "type": "text",
+ },
+ },
+ }
+ dash := models.Dashboard{
+ Title: "Testing ConnectLibraryPanelsForDashboard",
+ Data: simplejson.NewFromAny(dashJSON),
+ }
+ dashInDB := createDashboard(t, sc.sqlStore, sc.user, &dash, sc.folder.Id)
+
+ err = sc.service.ConnectLibraryPanelsForDashboard(sc.reqContext, dashInDB)
+ require.NoError(t, err)
+
+ elements, err := sc.elementService.GetElementsForDashboard(sc.reqContext, dashInDB.Id)
+ require.NoError(t, err)
+ require.Len(t, elements, 2)
+ require.Equal(t, sc.initialResult.Result.UID, elements[sc.initialResult.Result.UID].UID)
+ require.Equal(t, outsidePanel.UID, elements[outsidePanel.UID].UID)
+ })
+
scenarioWithLibraryPanel(t, "When an admin tries to store a dashboard with a library panel without uid, it should fail",
func(t *testing.T, sc scenarioContext) {
dashJSON := map[string]interface{}{
diff --git a/pkg/services/live/live.go b/pkg/services/live/live.go
index eabee22374f..ec3d2f3f215 100644
--- a/pkg/services/live/live.go
+++ b/pkg/services/live/live.go
@@ -6,12 +6,11 @@ import (
"fmt"
"net/http"
"net/url"
+ "os"
"strings"
"sync"
"time"
- "github.com/gobwas/glob"
-
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/api/routing"
@@ -28,6 +27,7 @@ import (
"github.com/grafana/grafana/pkg/services/live/liveplugin"
"github.com/grafana/grafana/pkg/services/live/managedstream"
"github.com/grafana/grafana/pkg/services/live/orgchannel"
+ "github.com/grafana/grafana/pkg/services/live/pipeline"
"github.com/grafana/grafana/pkg/services/live/pushws"
"github.com/grafana/grafana/pkg/services/live/runstream"
"github.com/grafana/grafana/pkg/services/live/survey"
@@ -37,6 +37,7 @@ import (
"github.com/grafana/grafana/pkg/util"
"github.com/centrifugal/centrifuge"
+ "github.com/gobwas/glob"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/live"
"gopkg.in/redis.v5"
@@ -47,15 +48,6 @@ var (
loggerCF = log.New("live.centrifuge")
)
-func NewGrafanaLive() *GrafanaLive {
- return &GrafanaLive{
- channels: make(map[string]models.ChannelHandler),
- GrafanaScope: CoreGrafanaScope{
- Features: make(map[string]models.ChannelHandlerFactory),
- },
- }
-}
-
// CoreGrafanaScope list of core features
type CoreGrafanaScope struct {
Features map[string]models.ChannelHandlerFactory
@@ -122,9 +114,6 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r
broker, err := centrifuge.NewRedisBroker(node, centrifuge.RedisBrokerConfig{
Prefix: "gf_live",
- // We are using Redis streams here for history. Require Redis >= 5.
- UseStreams: true,
-
// Use reasonably large expiration interval for stream meta key,
// much bigger than maximum HistoryLifetime value in Node config.
// This way stream meta data will expire, in some cases you may want
@@ -149,20 +138,7 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r
node.SetPresenceManager(presenceManager)
}
- g.contextGetter = liveplugin.NewContextGetter(g.PluginContextProvider)
- channelLocalPublisher := liveplugin.NewChannelLocalPublisher(node)
- numLocalSubscribersGetter := liveplugin.NewNumLocalSubscribersGetter(node)
- g.runStreamManager = runstream.NewManager(channelLocalPublisher, numLocalSubscribersGetter, g.contextGetter)
-
- // Initialize the main features
- dash := &features.DashboardHandler{
- Publisher: g.Publish,
- ClientCount: g.ClientCount,
- }
- g.storage = database.NewStorage(g.SQLStore, g.CacheService)
- g.GrafanaScope.Dashboards = dash
- g.GrafanaScope.Features["dashboard"] = dash
- g.GrafanaScope.Features["broadcast"] = features.NewBroadcastRunner(g.storage)
+ channelLocalPublisher := liveplugin.NewChannelLocalPublisher(node, nil)
var managedStreamRunner *managedstream.Runner
if g.IsHA() {
@@ -175,16 +151,58 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r
}
managedStreamRunner = managedstream.NewRunner(
g.Publish,
+ channelLocalPublisher,
managedstream.NewRedisFrameCache(redisClient),
)
} else {
managedStreamRunner = managedstream.NewRunner(
g.Publish,
+ channelLocalPublisher,
managedstream.NewMemoryFrameCache(),
)
}
g.ManagedStreamRunner = managedStreamRunner
+ if enabled := g.Cfg.FeatureToggles["live-pipeline"]; enabled {
+ var builder pipeline.RuleBuilder
+ if os.Getenv("GF_LIVE_DEV_BUILDER") != "" {
+ builder = &pipeline.DevRuleBuilder{
+ Node: node,
+ ManagedStream: g.ManagedStreamRunner,
+ FrameStorage: pipeline.NewFrameStorage(),
+ }
+ } else {
+ storage := &pipeline.FileStorage{}
+ g.channelRuleStorage = storage
+ builder = &pipeline.StorageRuleBuilder{
+ Node: node,
+ ManagedStream: g.ManagedStreamRunner,
+ FrameStorage: pipeline.NewFrameStorage(),
+ RuleStorage: storage,
+ }
+ }
+ channelRuleGetter := pipeline.NewCacheSegmentedTree(builder)
+ g.Pipeline, err = pipeline.New(channelRuleGetter)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ g.contextGetter = liveplugin.NewContextGetter(g.PluginContextProvider)
+ pipelinedChannelLocalPublisher := liveplugin.NewChannelLocalPublisher(node, g.Pipeline)
+ numLocalSubscribersGetter := liveplugin.NewNumLocalSubscribersGetter(node)
+ g.runStreamManager = runstream.NewManager(pipelinedChannelLocalPublisher, numLocalSubscribersGetter, g.contextGetter)
+
+ // Initialize the main features
+ dash := &features.DashboardHandler{
+ Publisher: g.Publish,
+ ClientCount: g.ClientCount,
+ }
+ g.storage = database.NewStorage(g.SQLStore, g.CacheService)
+ g.GrafanaScope.Dashboards = dash
+ g.GrafanaScope.Features["dashboard"] = dash
+ g.GrafanaScope.Features["broadcast"] = features.NewBroadcastRunner(g.storage)
+
g.surveyCaller = survey.NewCaller(managedStreamRunner, node)
err = g.surveyCaller.SetupHandlers()
if err != nil {
@@ -335,12 +353,19 @@ type GrafanaLive struct {
GrafanaScope CoreGrafanaScope
ManagedStreamRunner *managedstream.Runner
+ Pipeline *pipeline.Pipeline
+ channelRuleStorage pipeline.RuleStorage
contextGetter *liveplugin.ContextGetter
runStreamManager *runstream.Manager
storage *database.Storage
}
+type UsageStats struct {
+ NumClients int
+ NumUsers int
+}
+
func (g *GrafanaLive) getStreamPlugin(pluginID string) (backend.StreamHandler, error) {
plugin, ok := g.PluginManager.BackendPluginManager.Get(pluginID)
if !ok {
@@ -361,6 +386,10 @@ func (g *GrafanaLive) Run(ctx context.Context) error {
return nil
}
+func (g *GrafanaLive) ChannelRuleStorage() pipeline.RuleStorage {
+ return g.channelRuleStorage
+}
+
func getCheckOriginFunc(appURL *url.URL, originPatterns []string, originGlobs []glob.Glob) func(r *http.Request) bool {
return func(r *http.Request) bool {
origin := r.Header.Get("Origin")
@@ -686,7 +715,7 @@ func (g *GrafanaLive) handlePluginScope(_ *models.SignedInUser, namespace string
}
func (g *GrafanaLive) handleStreamScope(u *models.SignedInUser, namespace string) (models.ChannelHandlerFactory, error) {
- return g.ManagedStreamRunner.GetOrCreateStream(u.OrgId, namespace)
+ return g.ManagedStreamRunner.GetOrCreateStream(u.OrgId, live.ScopeStream, namespace)
}
func (g *GrafanaLive) handleDatasourceScope(user *models.SignedInUser, namespace string) (models.ChannelHandlerFactory, error) {
@@ -722,6 +751,13 @@ func (g *GrafanaLive) ClientCount(orgID int64, channel string) (int, error) {
return len(p.Presence), nil
}
+func (g *GrafanaLive) UsageStats() UsageStats {
+ clients := g.node.Hub().NumClients()
+ users := g.node.Hub().NumUsers()
+
+ return UsageStats{NumClients: clients, NumUsers: users}
+}
+
func (g *GrafanaLive) HandleHTTPPublish(ctx *models.ReqContext, cmd dtos.LivePublishCmd) response.Response {
addr, err := live.ParseChannel(cmd.Channel)
if err != nil {
@@ -791,6 +827,28 @@ func (g *GrafanaLive) HandleInfoHTTP(ctx *models.ReqContext) response.Response {
})
}
+// HandleChannelRulesListHTTP ...
+func (g *GrafanaLive) HandleChannelRulesListHTTP(c *models.ReqContext) response.Response {
+ result, err := g.channelRuleStorage.ListChannelRules(c.Req.Context(), c.OrgId)
+ if err != nil {
+ return response.Error(http.StatusInternalServerError, "Failed to get channel rules", err)
+ }
+ return response.JSON(http.StatusOK, util.DynMap{
+ "rules": result,
+ })
+}
+
+// HandleRemoteWriteBackendsListHTTP ...
+func (g *GrafanaLive) HandleRemoteWriteBackendsListHTTP(c *models.ReqContext) response.Response {
+ result, err := g.channelRuleStorage.ListRemoteWriteBackends(c.Req.Context(), c.OrgId)
+ if err != nil {
+ return response.Error(http.StatusInternalServerError, "Failed to get channel rules", err)
+ }
+ return response.JSON(http.StatusOK, util.DynMap{
+ "remoteWriteBackends": result,
+ })
+}
+
// Write to the standard log15 logger
func handleLog(msg centrifuge.LogEntry) {
arr := make([]interface{}, 0)
diff --git a/pkg/services/live/liveplugin/plugin.go b/pkg/services/live/liveplugin/plugin.go
index 3c7939d0d1a..bbfdcb7cafb 100644
--- a/pkg/services/live/liveplugin/plugin.go
+++ b/pkg/services/live/liveplugin/plugin.go
@@ -1,24 +1,42 @@
package liveplugin
import (
+ "context"
"fmt"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins/plugincontext"
+ "github.com/grafana/grafana/pkg/services/live/orgchannel"
+ "github.com/grafana/grafana/pkg/services/live/pipeline"
"github.com/centrifugal/centrifuge"
"github.com/grafana/grafana-plugin-sdk-go/backend"
)
type ChannelLocalPublisher struct {
- node *centrifuge.Node
+ node *centrifuge.Node
+ pipeline *pipeline.Pipeline
}
-func NewChannelLocalPublisher(node *centrifuge.Node) *ChannelLocalPublisher {
- return &ChannelLocalPublisher{node: node}
+func NewChannelLocalPublisher(node *centrifuge.Node, pipeline *pipeline.Pipeline) *ChannelLocalPublisher {
+ return &ChannelLocalPublisher{node: node, pipeline: pipeline}
}
func (p *ChannelLocalPublisher) PublishLocal(channel string, data []byte) error {
+ if p.pipeline != nil {
+ orgID, channelID, err := orgchannel.StripOrgID(channel)
+ if err != nil {
+ return err
+ }
+ ok, err := p.pipeline.ProcessInput(context.Background(), orgID, channelID, data)
+ if err != nil {
+ return err
+ }
+ if ok {
+ // if rule found – we are done here. If not - fall through and process as usual.
+ return nil
+ }
+ }
pub := ¢rifuge.Publication{
Data: data,
}
diff --git a/pkg/services/live/managedstream/cache.go b/pkg/services/live/managedstream/cache.go
index 9f84437efee..8ab810ce58b 100644
--- a/pkg/services/live/managedstream/cache.go
+++ b/pkg/services/live/managedstream/cache.go
@@ -10,7 +10,7 @@ import (
type FrameCache interface {
// GetActiveChannels returns active managed stream channels with JSON schema.
GetActiveChannels(orgID int64) (map[string]json.RawMessage, error)
- // GetFrame returns full JSON frame for a path.
+ // GetFrame returns full JSON frame for a channel in org.
GetFrame(orgID int64, channel string) (json.RawMessage, bool, error)
// Update updates frame cache and returns true if schema changed.
Update(orgID int64, channel string, frameJson data.FrameJSONCache) (bool, error)
diff --git a/pkg/services/live/managedstream/runner.go b/pkg/services/live/managedstream/runner.go
index 3069cdff022..609da93396b 100644
--- a/pkg/services/live/managedstream/runner.go
+++ b/pkg/services/live/managedstream/runner.go
@@ -8,6 +8,8 @@ import (
"sync"
"time"
+ "github.com/grafana/grafana/pkg/services/live/orgchannel"
+
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/live"
@@ -19,20 +21,39 @@ var (
logger = log.New("live.managed_stream")
)
-// Runner keeps ManagedStream per streamID.
+// If message comes from a plugin:
+// * it's simply sent to local subscribers without any additional steps
+// * if there is RULE then may be processed in some way
+// * important to keep a message in the original channel
+// * client subscribed to ds//xxx
+//
+// What we want to build:
+// * Stream scope not hardcoded and determined by the caller
+// * So it's possible to use managed stream from plugins
+// * The problem is HA – at moment several plugins on different nodes publish same messages
+// * Can use in-memory managed stream for plugins with local subscribers publish, use HA-managed stream for HTTP/WS
+// * Eventually maintain a single connection with a plugin over a channel leader selection.
+
+// Runner keeps NamespaceStream per namespace.
type Runner struct {
- mu sync.RWMutex
- streams map[int64]map[string]*ManagedStream
- publisher models.ChannelPublisher
- frameCache FrameCache
+ mu sync.RWMutex
+ streams map[int64]map[string]*NamespaceStream
+ publisher models.ChannelPublisher
+ localPublisher LocalPublisher
+ frameCache FrameCache
+}
+
+type LocalPublisher interface {
+ PublishLocal(channel string, data []byte) error
}
// NewRunner creates new Runner.
-func NewRunner(publisher models.ChannelPublisher, frameCache FrameCache) *Runner {
+func NewRunner(publisher models.ChannelPublisher, localPublisher LocalPublisher, frameCache FrameCache) *Runner {
return &Runner{
- publisher: publisher,
- streams: map[int64]map[string]*ManagedStream{},
- frameCache: frameCache,
+ publisher: publisher,
+ localPublisher: localPublisher,
+ streams: map[int64]map[string]*NamespaceStream{},
+ frameCache: frameCache,
}
}
@@ -49,7 +70,8 @@ func (r *Runner) GetManagedChannels(orgID int64) ([]*ManagedChannel, error) {
}
// Enrich with minute rate.
channel, _ := live.ParseChannel(managedChannel.Channel)
- namespaceStream, ok := r.streams[orgID][channel.Namespace]
+ prefix := channel.Scope + "/" + channel.Namespace
+ namespaceStream, ok := r.streams[orgID][prefix]
if ok {
managedChannel.MinuteRate = namespaceStream.minuteRate(channel.Path)
}
@@ -86,46 +108,34 @@ func (r *Runner) GetManagedChannels(orgID int64) ([]*ManagedChannel, error) {
return channels, nil
}
-// Streams returns a map of active managed streams (per streamID).
-func (r *Runner) Streams(orgID int64) map[string]*ManagedStream {
- r.mu.RLock()
- defer r.mu.RUnlock()
- if _, ok := r.streams[orgID]; !ok {
- return map[string]*ManagedStream{}
- }
- streams := make(map[string]*ManagedStream, len(r.streams[orgID]))
- for k, v := range r.streams[orgID] {
- streams[k] = v
- }
- return streams
-}
-
// GetOrCreateStream -- for now this will create new manager for each key.
// Eventually, the stream behavior will need to be configured explicitly
-func (r *Runner) GetOrCreateStream(orgID int64, streamID string) (*ManagedStream, error) {
+func (r *Runner) GetOrCreateStream(orgID int64, scope string, namespace string) (*NamespaceStream, error) {
r.mu.Lock()
defer r.mu.Unlock()
_, ok := r.streams[orgID]
if !ok {
- r.streams[orgID] = map[string]*ManagedStream{}
+ r.streams[orgID] = map[string]*NamespaceStream{}
}
- s, ok := r.streams[orgID][streamID]
+ prefix := scope + "/" + namespace
+ s, ok := r.streams[orgID][prefix]
if !ok {
- s = NewManagedStream(streamID, orgID, r.publisher, r.frameCache)
- r.streams[orgID][streamID] = s
+ s = NewNamespaceStream(orgID, scope, namespace, r.publisher, r.localPublisher, r.frameCache)
+ r.streams[orgID][prefix] = s
}
return s, nil
}
-// ManagedStream holds the state of a managed stream.
-type ManagedStream struct {
- id string
- orgID int64
- start time.Time
- publisher models.ChannelPublisher
- frameCache FrameCache
- rateMu sync.RWMutex
- rates map[string][60]rateEntry
+// NamespaceStream holds the state of a managed stream.
+type NamespaceStream struct {
+ orgID int64
+ scope string
+ namespace string
+ publisher models.ChannelPublisher
+ localPublisher LocalPublisher
+ frameCache FrameCache
+ rateMu sync.RWMutex
+ rates map[string][60]rateEntry
}
type rateEntry struct {
@@ -133,18 +143,6 @@ type rateEntry struct {
count int32
}
-// NewManagedStream creates new ManagedStream.
-func NewManagedStream(id string, orgID int64, publisher models.ChannelPublisher, schemaUpdater FrameCache) *ManagedStream {
- return &ManagedStream{
- id: id,
- orgID: orgID,
- start: time.Now(),
- publisher: publisher,
- frameCache: schemaUpdater,
- rates: map[string][60]rateEntry{},
- }
-}
-
// ManagedChannel represents a managed stream.
type ManagedChannel struct {
Channel string `json:"channel"`
@@ -152,16 +150,30 @@ type ManagedChannel struct {
Data json.RawMessage `json:"data"`
}
+// NewNamespaceStream creates new NamespaceStream.
+func NewNamespaceStream(orgID int64, scope string, namespace string, publisher models.ChannelPublisher, localPublisher LocalPublisher, schemaUpdater FrameCache) *NamespaceStream {
+ return &NamespaceStream{
+ orgID: orgID,
+ scope: scope,
+ namespace: namespace,
+ publisher: publisher,
+ localPublisher: localPublisher,
+ frameCache: schemaUpdater,
+ rates: map[string][60]rateEntry{},
+ }
+}
+
// Push sends frame to the stream and saves it for later retrieval by subscribers.
-// unstableSchema flag can be set to disable schema caching for a path.
-func (s *ManagedStream) Push(path string, frame *data.Frame) error {
+// * Saves the entire frame to cache.
+// * If schema has been changed sends entire frame to channel, otherwise only data.
+func (s *NamespaceStream) Push(path string, frame *data.Frame) error {
jsonFrameCache, err := data.FrameToJSONCache(frame)
if err != nil {
return err
}
// The channel this will be posted into.
- channel := live.Channel{Scope: live.ScopeStream, Namespace: s.id, Path: path}.String()
+ channel := live.Channel{Scope: s.scope, Namespace: s.namespace, Path: path}.String()
isUpdated, err := s.frameCache.Update(s.orgID, channel, jsonFrameCache)
if err != nil {
@@ -179,10 +191,13 @@ func (s *ManagedStream) Push(path string, frame *data.Frame) error {
logger.Debug("Publish data to channel", "channel", channel, "dataLength", len(frameJSON))
s.incRate(path, time.Now().Unix())
+ if s.scope == live.ScopeDatasource || s.scope == live.ScopePlugin {
+ return s.localPublisher.PublishLocal(orgchannel.PrependOrgID(s.orgID, channel), frameJSON)
+ }
return s.publisher(s.orgID, channel, frameJSON)
}
-func (s *ManagedStream) incRate(path string, nowUnix int64) {
+func (s *NamespaceStream) incRate(path string, nowUnix int64) {
s.rateMu.Lock()
pathRate, ok := s.rates[path]
if !ok {
@@ -199,7 +214,7 @@ func (s *ManagedStream) incRate(path string, nowUnix int64) {
s.rateMu.Unlock()
}
-func (s *ManagedStream) minuteRate(path string) int64 {
+func (s *NamespaceStream) minuteRate(path string) int64 {
var total int64
s.rateMu.RLock()
defer s.rateMu.RUnlock()
@@ -215,11 +230,11 @@ func (s *ManagedStream) minuteRate(path string) int64 {
return total
}
-func (s *ManagedStream) GetHandlerForPath(_ string) (models.ChannelHandler, error) {
+func (s *NamespaceStream) GetHandlerForPath(_ string) (models.ChannelHandler, error) {
return s, nil
}
-func (s *ManagedStream) OnSubscribe(_ context.Context, u *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) {
+func (s *NamespaceStream) OnSubscribe(_ context.Context, u *models.SignedInUser, e models.SubscribeEvent) (models.SubscribeReply, backend.SubscribeStreamStatus, error) {
reply := models.SubscribeReply{}
frameJSON, ok, err := s.frameCache.GetFrame(u.OrgId, e.Channel)
if err != nil {
@@ -231,6 +246,6 @@ func (s *ManagedStream) OnSubscribe(_ context.Context, u *models.SignedInUser, e
return reply, backend.SubscribeStreamStatusOK, nil
}
-func (s *ManagedStream) OnPublish(_ context.Context, _ *models.SignedInUser, _ models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
+func (s *NamespaceStream) OnPublish(_ context.Context, _ *models.SignedInUser, _ models.PublishEvent) (models.PublishReply, backend.PublishStreamStatus, error) {
return models.PublishReply{}, backend.PublishStreamStatusPermissionDenied, nil
}
diff --git a/pkg/services/live/managedstream/runner_test.go b/pkg/services/live/managedstream/runner_test.go
index 69153a5a9b4..66f0347cca6 100644
--- a/pkg/services/live/managedstream/runner_test.go
+++ b/pkg/services/live/managedstream/runner_test.go
@@ -18,13 +18,13 @@ func (p *testPublisher) publish(_ int64, _ string, _ []byte) error {
func TestNewManagedStream(t *testing.T) {
publisher := &testPublisher{t: t}
- c := NewManagedStream("a", 1, publisher.publish, NewMemoryFrameCache())
+ c := NewNamespaceStream(1, "stream", "a", publisher.publish, nil, NewMemoryFrameCache())
require.NotNil(t, c)
}
func TestManagedStreamMinuteRate(t *testing.T) {
publisher := &testPublisher{t: t}
- c := NewManagedStream("a", 1, publisher.publish, NewMemoryFrameCache())
+ c := NewNamespaceStream(1, "stream", "a", publisher.publish, nil, NewMemoryFrameCache())
require.NotNil(t, c)
c.incRate("test1", time.Now().Unix())
@@ -47,10 +47,10 @@ func TestManagedStreamMinuteRate(t *testing.T) {
func TestGetManagedStreams(t *testing.T) {
publisher := &testPublisher{t: t}
frameCache := NewMemoryFrameCache()
- runner := NewRunner(publisher.publish, frameCache)
- s1, err := runner.GetOrCreateStream(1, "test1")
+ runner := NewRunner(publisher.publish, nil, frameCache)
+ s1, err := runner.GetOrCreateStream(1, "stream", "test1")
require.NoError(t, err)
- s2, err := runner.GetOrCreateStream(1, "test2")
+ s2, err := runner.GetOrCreateStream(1, "stream", "test2")
require.NoError(t, err)
managedChannels, err := runner.GetManagedChannels(1)
@@ -74,7 +74,7 @@ func TestGetManagedStreams(t *testing.T) {
require.Equal(t, "stream/test2/cpu1", managedChannels[5].Channel)
// Different org.
- s3, err := runner.GetOrCreateStream(2, "test1")
+ s3, err := runner.GetOrCreateStream(2, "stream", "test1")
require.NoError(t, err)
err = s3.Push("cpu1", data.NewFrame("cpu1"))
require.NoError(t, err)
diff --git a/pkg/services/live/pipeline/condition_checker.go b/pkg/services/live/pipeline/condition_checker.go
new file mode 100644
index 00000000000..46d4607612d
--- /dev/null
+++ b/pkg/services/live/pipeline/condition_checker.go
@@ -0,0 +1,12 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// ConditionChecker checks conditions in context of data.Frame being processed.
+type ConditionChecker interface {
+ CheckCondition(ctx context.Context, frame *data.Frame) (bool, error)
+}
diff --git a/pkg/services/live/pipeline/condition_checker_multiple.go b/pkg/services/live/pipeline/condition_checker_multiple.go
new file mode 100644
index 00000000000..dee496331df
--- /dev/null
+++ b/pkg/services/live/pipeline/condition_checker_multiple.go
@@ -0,0 +1,45 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// ConditionType represents multiple condition operator type.
+type ConditionType string
+
+const (
+ ConditionAll ConditionType = "all"
+ ConditionAny ConditionType = "any"
+)
+
+// MultipleConditionChecker can check multiple conditions according to ConditionType.
+type MultipleConditionChecker struct {
+ Type ConditionType
+ Conditions []ConditionChecker
+}
+
+func (m MultipleConditionChecker) CheckCondition(ctx context.Context, frame *data.Frame) (bool, error) {
+ for _, c := range m.Conditions {
+ ok, err := c.CheckCondition(ctx, frame)
+ if err != nil {
+ return false, err
+ }
+ if ok && m.Type == ConditionAny {
+ return true, nil
+ }
+ if !ok && m.Type == ConditionAll {
+ return false, nil
+ }
+ }
+ if m.Type == ConditionAny {
+ return false, nil
+ }
+ return true, nil
+}
+
+// NewMultipleConditionChecker creates new MultipleConditionChecker.
+func NewMultipleConditionChecker(conditionType ConditionType, conditions ...ConditionChecker) *MultipleConditionChecker {
+ return &MultipleConditionChecker{Type: conditionType, Conditions: conditions}
+}
diff --git a/pkg/services/live/pipeline/condition_number_compare.go b/pkg/services/live/pipeline/condition_number_compare.go
new file mode 100644
index 00000000000..429415c5b30
--- /dev/null
+++ b/pkg/services/live/pipeline/condition_number_compare.go
@@ -0,0 +1,64 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// NumberCompareCondition can compare numbers.
+type NumberCompareCondition struct {
+ FieldName string
+ Op NumberCompareOp
+ Value float64
+}
+
+// NumberCompareOp is an comparison operator.
+type NumberCompareOp string
+
+// Known NumberCompareOp types.
+const (
+ NumberCompareOpLt NumberCompareOp = "lt"
+ NumberCompareOpGt NumberCompareOp = "gt"
+ NumberCompareOpLte NumberCompareOp = "lte"
+ NumberCompareOpGte NumberCompareOp = "gte"
+ NumberCompareOpEq NumberCompareOp = "eq"
+ NumberCompareOpNe NumberCompareOp = "ne"
+)
+
+func (f NumberCompareCondition) CheckCondition(_ context.Context, frame *data.Frame) (bool, error) {
+ for _, field := range frame.Fields {
+ // TODO: support other numeric types.
+ if field.Name == f.FieldName && (field.Type() == data.FieldTypeNullableFloat64) {
+ value, ok := field.At(0).(*float64)
+ if !ok {
+ return false, fmt.Errorf("unexpected value type: %T", field.At(0))
+ }
+ if value == nil {
+ return false, nil
+ }
+ switch f.Op {
+ case NumberCompareOpGt:
+ return *value > f.Value, nil
+ case NumberCompareOpGte:
+ return *value >= f.Value, nil
+ case NumberCompareOpLte:
+ return *value <= f.Value, nil
+ case NumberCompareOpLt:
+ return *value < f.Value, nil
+ case NumberCompareOpEq:
+ return *value == f.Value, nil
+ case NumberCompareOpNe:
+ return *value != f.Value, nil
+ default:
+ return false, fmt.Errorf("unknown comparison operator: %s", f.Op)
+ }
+ }
+ }
+ return false, nil
+}
+
+func NewNumberCompareCondition(fieldName string, op NumberCompareOp, value float64) *NumberCompareCondition {
+ return &NumberCompareCondition{FieldName: fieldName, Op: op, Value: value}
+}
diff --git a/pkg/services/live/pipeline/config.go b/pkg/services/live/pipeline/config.go
new file mode 100644
index 00000000000..d5efa9142e1
--- /dev/null
+++ b/pkg/services/live/pipeline/config.go
@@ -0,0 +1,321 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/grafana/grafana/pkg/services/live/managedstream"
+
+ "github.com/centrifugal/centrifuge"
+)
+
+type JsonAutoSettings struct{}
+
+type ConverterConfig struct {
+ Type string `json:"type"`
+ AutoJsonConverterConfig *AutoJsonConverterConfig `json:"jsonAuto,omitempty"`
+ ExactJsonConverterConfig *ExactJsonConverterConfig `json:"jsonExact,omitempty"`
+ AutoInfluxConverterConfig *AutoInfluxConverterConfig `json:"influxAuto,omitempty"`
+ JsonFrameConverterConfig *JsonFrameConverterConfig `json:"jsonFrame,omitempty"`
+}
+
+type ProcessorConfig struct {
+ Type string `json:"type"`
+ DropFieldsProcessorConfig *DropFieldsProcessorConfig `json:"dropFields,omitempty"`
+ KeepFieldsProcessorConfig *KeepFieldsProcessorConfig `json:"keepFields,omitempty"`
+ MultipleProcessorConfig *MultipleProcessorConfig `json:"multiple,omitempty"`
+}
+
+type MultipleProcessorConfig struct {
+ Processors []ProcessorConfig `json:"processors"`
+}
+
+type MultipleOutputterConfig struct {
+ Outputters []OutputterConfig `json:"outputs"`
+}
+
+type ManagedStreamOutputConfig struct{}
+
+type ConditionalOutputConfig struct {
+ Condition *ConditionCheckerConfig `json:"condition"`
+ Outputter *OutputterConfig `json:"output"`
+}
+
+type RemoteWriteOutputConfig struct {
+ UID string `json:"uid"`
+}
+
+type OutputterConfig struct {
+ Type string `json:"type"`
+ ManagedStreamConfig *ManagedStreamOutputConfig `json:"managedStream,omitempty"`
+ MultipleOutputterConfig *MultipleOutputterConfig `json:"multiple,omitempty"`
+ RedirectOutputConfig *RedirectOutputConfig `json:"redirect,omitempty"`
+ ConditionalOutputConfig *ConditionalOutputConfig `json:"conditional,omitempty"`
+ ThresholdOutputConfig *ThresholdOutputConfig `json:"threshold,omitempty"`
+ RemoteWriteOutputConfig *RemoteWriteOutputConfig `json:"remoteWrite,omitempty"`
+ ChangeLogOutputConfig *ChangeLogOutputConfig `json:"changeLog,omitempty"`
+}
+
+type ChannelRuleSettings struct {
+ Converter *ConverterConfig `json:"converter,omitempty"`
+ Processor *ProcessorConfig `json:"processor,omitempty"`
+ Outputter *OutputterConfig `json:"output,omitempty"`
+}
+
+type ChannelRule struct {
+ OrgId int64 `json:"-"`
+ Pattern string `json:"pattern"`
+ Settings ChannelRuleSettings `json:"settings"`
+}
+
+type RemoteWriteBackend struct {
+ OrgId int64 `json:"-"`
+ UID string `json:"uid"`
+ Settings *RemoteWriteConfig `json:"settings"`
+}
+
+type RemoteWriteBackends struct {
+ Backends []RemoteWriteBackend `json:"remoteWriteBackends"`
+}
+
+type ChannelRules struct {
+ Rules []ChannelRule `json:"rules"`
+}
+
+type MultipleConditionCheckerConfig struct {
+ Type ConditionType `json:"type"`
+ Conditions []ConditionCheckerConfig `json:"conditions"`
+}
+
+type NumberCompareConditionConfig struct {
+ FieldName string `json:"fieldName"`
+ Op NumberCompareOp `json:"op"`
+ Value float64 `json:"value"`
+}
+
+type ConditionCheckerConfig struct {
+ Type string `json:"type"`
+ MultipleConditionCheckerConfig *MultipleConditionCheckerConfig `json:"multiple,omitempty"`
+ NumberCompareConditionConfig *NumberCompareConditionConfig `json:"numberCompare,omitempty"`
+}
+
+type RuleStorage interface {
+ ListRemoteWriteBackends(_ context.Context, orgID int64) ([]RemoteWriteBackend, error)
+ ListChannelRules(_ context.Context, orgID int64) ([]ChannelRule, error)
+}
+
+type StorageRuleBuilder struct {
+ Node *centrifuge.Node
+ ManagedStream *managedstream.Runner
+ FrameStorage *FrameStorage
+ RuleStorage RuleStorage
+}
+
+func (f *StorageRuleBuilder) extractConverter(config *ConverterConfig) (Converter, error) {
+ if config == nil {
+ return nil, nil
+ }
+ missingConfiguration := fmt.Errorf("missing configuration for %s", config.Type)
+ switch config.Type {
+ case "jsonAuto":
+ if config.AutoJsonConverterConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewAutoJsonConverter(*config.AutoJsonConverterConfig), nil
+ case "jsonExact":
+ if config.ExactJsonConverterConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewExactJsonConverter(*config.ExactJsonConverterConfig), nil
+ case "jsonFrame":
+ if config.JsonFrameConverterConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewJsonFrameConverter(*config.JsonFrameConverterConfig), nil
+ case "influxAuto":
+ if config.AutoInfluxConverterConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewAutoInfluxConverter(*config.AutoInfluxConverterConfig), nil
+ default:
+ return nil, fmt.Errorf("unknown converter type: %s", config.Type)
+ }
+}
+
+func (f *StorageRuleBuilder) extractProcessor(config *ProcessorConfig) (Processor, error) {
+ if config == nil {
+ return nil, nil
+ }
+ missingConfiguration := fmt.Errorf("missing configuration for %s", config.Type)
+ switch config.Type {
+ case "dropFields":
+ if config.DropFieldsProcessorConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewDropFieldsProcessor(*config.DropFieldsProcessorConfig), nil
+ case "keepFields":
+ if config.KeepFieldsProcessorConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewKeepFieldsProcessor(*config.KeepFieldsProcessorConfig), nil
+ case "multiple":
+ if config.MultipleProcessorConfig == nil {
+ return nil, missingConfiguration
+ }
+ var processors []Processor
+ for _, outConf := range config.MultipleProcessorConfig.Processors {
+ out := outConf
+ proc, err := f.extractProcessor(&out)
+ if err != nil {
+ return nil, err
+ }
+ processors = append(processors, proc)
+ }
+ return NewMultipleProcessor(processors...), nil
+ default:
+ return nil, fmt.Errorf("unknown processor type: %s", config.Type)
+ }
+}
+
+func (f *StorageRuleBuilder) extractConditionChecker(config *ConditionCheckerConfig) (ConditionChecker, error) {
+ if config == nil {
+ return nil, nil
+ }
+ missingConfiguration := fmt.Errorf("missing configuration for %s", config.Type)
+ switch config.Type {
+ case "numberCompare":
+ if config.NumberCompareConditionConfig == nil {
+ return nil, missingConfiguration
+ }
+ c := *config.NumberCompareConditionConfig
+ return NewNumberCompareCondition(c.FieldName, c.Op, c.Value), nil
+ case "multiple":
+ var conditions []ConditionChecker
+ if config.MultipleConditionCheckerConfig == nil {
+ return nil, missingConfiguration
+ }
+ for _, outConf := range config.MultipleConditionCheckerConfig.Conditions {
+ out := outConf
+ cond, err := f.extractConditionChecker(&out)
+ if err != nil {
+ return nil, err
+ }
+ conditions = append(conditions, cond)
+ }
+ return NewMultipleConditionChecker(config.MultipleConditionCheckerConfig.Type, conditions...), nil
+ default:
+ return nil, fmt.Errorf("unknown condition type: %s", config.Type)
+ }
+}
+
+func (f *StorageRuleBuilder) extractOutputter(config *OutputterConfig, remoteWriteBackends []RemoteWriteBackend) (Outputter, error) {
+ if config == nil {
+ return nil, nil
+ }
+ missingConfiguration := fmt.Errorf("missing configuration for %s", config.Type)
+ switch config.Type {
+ case "redirect":
+ if config.RedirectOutputConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewRedirectOutput(*config.RedirectOutputConfig), nil
+ case "multiple":
+ if config.MultipleOutputterConfig == nil {
+ return nil, missingConfiguration
+ }
+ var outputters []Outputter
+ for _, outConf := range config.MultipleOutputterConfig.Outputters {
+ out := outConf
+ outputter, err := f.extractOutputter(&out, remoteWriteBackends)
+ if err != nil {
+ return nil, err
+ }
+ outputters = append(outputters, outputter)
+ }
+ return NewMultipleOutput(outputters...), nil
+ case "managedStream":
+ return NewManagedStreamOutput(f.ManagedStream), nil
+ case "localSubscribers":
+ return NewLocalSubscribersOutput(f.Node), nil
+ case "conditional":
+ if config.ConditionalOutputConfig == nil {
+ return nil, missingConfiguration
+ }
+ condition, err := f.extractConditionChecker(config.ConditionalOutputConfig.Condition)
+ if err != nil {
+ return nil, err
+ }
+ outputter, err := f.extractOutputter(config.ConditionalOutputConfig.Outputter, remoteWriteBackends)
+ if err != nil {
+ return nil, err
+ }
+ return NewConditionalOutput(condition, outputter), nil
+ case "threshold":
+ if config.ThresholdOutputConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewThresholdOutput(f.FrameStorage, *config.ThresholdOutputConfig), nil
+ case "remoteWrite":
+ if config.RemoteWriteOutputConfig == nil {
+ return nil, missingConfiguration
+ }
+ remoteWriteConfig, ok := f.getRemoteWriteConfig(config.RemoteWriteOutputConfig.UID, remoteWriteBackends)
+ if !ok {
+ return nil, fmt.Errorf("unknown remote write backend uid: %s", config.RemoteWriteOutputConfig.UID)
+ }
+ return NewRemoteWriteOutput(*remoteWriteConfig), nil
+ case "changeLog":
+ if config.ChangeLogOutputConfig == nil {
+ return nil, missingConfiguration
+ }
+ return NewChangeLogOutput(f.FrameStorage, *config.ChangeLogOutputConfig), nil
+ default:
+ return nil, fmt.Errorf("unknown output type: %s", config.Type)
+ }
+}
+
+func (f *StorageRuleBuilder) getRemoteWriteConfig(uid string, remoteWriteBackends []RemoteWriteBackend) (*RemoteWriteConfig, bool) {
+ for _, rwb := range remoteWriteBackends {
+ if rwb.UID == uid {
+ return rwb.Settings, true
+ }
+ }
+ return nil, false
+}
+
+func (f *StorageRuleBuilder) BuildRules(ctx context.Context, orgID int64) ([]*LiveChannelRule, error) {
+ channelRules, err := f.RuleStorage.ListChannelRules(ctx, orgID)
+ if err != nil {
+ return nil, err
+ }
+
+ remoteWriteBackends, err := f.RuleStorage.ListRemoteWriteBackends(ctx, orgID)
+ if err != nil {
+ return nil, err
+ }
+
+ var rules []*LiveChannelRule
+
+ for _, ruleConfig := range channelRules {
+ rule := &LiveChannelRule{
+ OrgId: orgID,
+ Pattern: ruleConfig.Pattern,
+ }
+ var err error
+ rule.Converter, err = f.extractConverter(ruleConfig.Settings.Converter)
+ if err != nil {
+ return nil, err
+ }
+ rule.Processor, err = f.extractProcessor(ruleConfig.Settings.Processor)
+ if err != nil {
+ return nil, err
+ }
+ rule.Outputter, err = f.extractOutputter(ruleConfig.Settings.Outputter, remoteWriteBackends)
+ if err != nil {
+ return nil, err
+ }
+ rules = append(rules, rule)
+ }
+
+ return rules, nil
+}
diff --git a/pkg/services/live/pipeline/converter_influx_auto.go b/pkg/services/live/pipeline/converter_influx_auto.go
new file mode 100644
index 00000000000..810751fee6d
--- /dev/null
+++ b/pkg/services/live/pipeline/converter_influx_auto.go
@@ -0,0 +1,38 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana/pkg/services/live/convert"
+)
+
+type AutoInfluxConverterConfig struct {
+ FrameFormat string `json:"frameFormat"`
+}
+
+// AutoInfluxConverter decodes Influx line protocol input and transforms it
+// to several ChannelFrame objects where Channel is constructed from original
+// channel + / + .
+type AutoInfluxConverter struct {
+ config AutoInfluxConverterConfig
+ converter *convert.Converter
+}
+
+func NewAutoInfluxConverter(config AutoInfluxConverterConfig) *AutoInfluxConverter {
+ return &AutoInfluxConverter{config: config, converter: convert.NewConverter()}
+}
+
+func (i AutoInfluxConverter) Convert(_ context.Context, vars Vars, body []byte) ([]*ChannelFrame, error) {
+ frameWrappers, err := i.converter.Convert(body, i.config.FrameFormat)
+ if err != nil {
+ return nil, err
+ }
+ channelFrames := make([]*ChannelFrame, 0, len(frameWrappers))
+ for _, fw := range frameWrappers {
+ channelFrames = append(channelFrames, &ChannelFrame{
+ Channel: vars.Channel + "/" + fw.Key(),
+ Frame: fw.Frame(),
+ })
+ }
+ return channelFrames, nil
+}
diff --git a/pkg/services/live/pipeline/converter_json_auto.go b/pkg/services/live/pipeline/converter_json_auto.go
new file mode 100644
index 00000000000..0e16fd9820a
--- /dev/null
+++ b/pkg/services/live/pipeline/converter_json_auto.go
@@ -0,0 +1,39 @@
+package pipeline
+
+import (
+ "context"
+ "time"
+)
+
+type AutoJsonConverterConfig struct {
+ FieldTips map[string]Field `json:"fieldTips"`
+}
+
+type AutoJsonConverter struct {
+ config AutoJsonConverterConfig
+ nowTimeFunc func() time.Time
+}
+
+func NewAutoJsonConverter(c AutoJsonConverterConfig) *AutoJsonConverter {
+ return &AutoJsonConverter{config: c}
+}
+
+// Automatic conversion works this way:
+// * Time added automatically
+// * Nulls dropped
+// To preserve nulls we need FieldTips from a user.
+// Custom time can be injected on Processor stage theoretically.
+// Custom labels can be injected on Processor stage theoretically.
+func (c *AutoJsonConverter) Convert(_ context.Context, vars Vars, body []byte) ([]*ChannelFrame, error) {
+ nowTimeFunc := c.nowTimeFunc
+ if nowTimeFunc == nil {
+ nowTimeFunc = time.Now
+ }
+ frame, err := jsonDocToFrame(vars.Path, body, c.config.FieldTips, nowTimeFunc)
+ if err != nil {
+ return nil, err
+ }
+ return []*ChannelFrame{
+ {Channel: "", Frame: frame},
+ }, nil
+}
diff --git a/pkg/services/live/pipeline/converter_json_auto_test.go b/pkg/services/live/pipeline/converter_json_auto_test.go
new file mode 100644
index 00000000000..f74410f893d
--- /dev/null
+++ b/pkg/services/live/pipeline/converter_json_auto_test.go
@@ -0,0 +1,52 @@
+package pipeline
+
+import (
+ "context"
+ "flag"
+ "io/ioutil"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/experimental"
+ "github.com/stretchr/testify/require"
+)
+
+var update = flag.Bool("update", false, "update golden files")
+
+func loadTestJson(tb testing.TB, file string) []byte {
+ tb.Helper()
+ // Safe to disable, this is a test.
+ // nolint:gosec
+ content, err := ioutil.ReadFile(filepath.Join("testdata", file+".json"))
+ require.NoError(tb, err, "expected to be able to read file")
+ require.True(tb, len(content) > 0)
+ return content
+}
+
+func checkAutoConversion(tb testing.TB, file string) *backend.DataResponse {
+ tb.Helper()
+ content := loadTestJson(tb, file)
+
+ converter := NewAutoJsonConverter(AutoJsonConverterConfig{})
+ converter.nowTimeFunc = func() time.Time {
+ return time.Date(2021, 01, 01, 12, 12, 12, 0, time.UTC)
+ }
+ channelFrames, err := converter.Convert(context.Background(), Vars{}, content)
+ require.NoError(tb, err)
+
+ dr := &backend.DataResponse{}
+ for _, cf := range channelFrames {
+ require.Empty(tb, cf.Channel)
+ dr.Frames = append(dr.Frames, cf.Frame)
+ }
+
+ err = experimental.CheckGoldenDataResponse(filepath.Join("testdata", file+".golden.txt"), dr, *update)
+ require.NoError(tb, err)
+ return dr
+}
+
+func TestAutoJsonConverter_Convert(t *testing.T) {
+ checkAutoConversion(t, "json_auto")
+}
diff --git a/pkg/services/live/pipeline/converter_json_exact.go b/pkg/services/live/pipeline/converter_json_exact.go
new file mode 100644
index 00000000000..d48ccc0e5d0
--- /dev/null
+++ b/pkg/services/live/pipeline/converter_json_exact.go
@@ -0,0 +1,210 @@
+package pipeline
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/spyzhov/ajson"
+)
+
+type ExactJsonConverterConfig struct {
+ Fields []Field `json:"fields"`
+}
+
+// ExactJsonConverter can convert JSON to a single data.Frame according to
+// user-defined field configuration and value extraction rules.
+type ExactJsonConverter struct {
+ config ExactJsonConverterConfig
+ nowTimeFunc func() time.Time
+}
+
+func NewExactJsonConverter(c ExactJsonConverterConfig) *ExactJsonConverter {
+ return &ExactJsonConverter{config: c}
+}
+
+func (c *ExactJsonConverter) Convert(_ context.Context, vars Vars, body []byte) ([]*ChannelFrame, error) {
+ //obj, err := oj.Parse(body)
+ //if err != nil {
+ // return nil, err
+ //}
+
+ var fields []*data.Field
+
+ var initGojaOnce sync.Once
+ var gojaRuntime *gojaRuntime
+
+ for _, f := range c.config.Fields {
+ field := data.NewFieldFromFieldType(f.Type, 1)
+ field.Name = f.Name
+ field.Config = f.Config
+
+ if strings.HasPrefix(f.Value, "$") {
+ // JSON path.
+ nodes, err := ajson.JSONPath(body, f.Value)
+ if err != nil {
+ return nil, err
+ }
+ if len(nodes) == 0 {
+ field.Set(0, nil)
+ } else if len(nodes) == 1 {
+ val, err := nodes[0].Value()
+ if err != nil {
+ return nil, err
+ }
+ switch f.Type {
+ case data.FieldTypeNullableFloat64:
+ if val == nil {
+ field.Set(0, nil)
+ } else {
+ switch v := val.(type) {
+ case float64:
+ field.SetConcrete(0, v)
+ case int64:
+ field.SetConcrete(0, float64(v))
+ default:
+ return nil, errors.New("malformed float64 type for: " + f.Name)
+ }
+ }
+ case data.FieldTypeNullableString:
+ v, ok := val.(string)
+ if !ok {
+ return nil, errors.New("malformed string type")
+ }
+ field.SetConcrete(0, v)
+ default:
+ return nil, fmt.Errorf("unsupported field type: %s (%s)", f.Type, f.Name)
+ }
+ } else {
+ return nil, errors.New("too many values")
+ }
+ //x, err := jp.ParseString(f.Value[1:])
+ //if err != nil {
+ // return nil, err
+ //}
+ //value := x.Get(obj)
+ //if len(value) == 0 {
+ // field.Set(0, nil)
+ //} else if len(value) == 1 {
+ // val := value[0]
+ // switch f.Type {
+ // case data.FieldTypeNullableFloat64:
+ // if val == nil {
+ // field.Set(0, nil)
+ // } else {
+ // switch v := val.(type) {
+ // case float64:
+ // field.SetConcrete(0, v)
+ // case int64:
+ // field.SetConcrete(0, float64(v))
+ // default:
+ // return nil, errors.New("malformed float64 type for: " + f.Name)
+ // }
+ // }
+ // case data.FieldTypeNullableString:
+ // v, ok := val.(string)
+ // if !ok {
+ // return nil, errors.New("malformed string type")
+ // }
+ // field.SetConcrete(0, v)
+ // default:
+ // return nil, fmt.Errorf("unsupported field type: %s (%s)", f.Type, f.Name)
+ // }
+ //} else {
+ // return nil, errors.New("too many values")
+ //}
+ } else if strings.HasPrefix(f.Value, "{") {
+ // Goja script.
+ script := strings.Trim(f.Value, "{}")
+ var err error
+ initGojaOnce.Do(func() {
+ gojaRuntime, err = getRuntime(body)
+ })
+ if err != nil {
+ return nil, err
+ }
+ switch f.Type {
+ case data.FieldTypeNullableBool:
+ v, err := gojaRuntime.getBool(script)
+ if err != nil {
+ return nil, err
+ }
+ field.SetConcrete(0, v)
+ case data.FieldTypeNullableFloat64:
+ v, err := gojaRuntime.getFloat64(script)
+ if err != nil {
+ return nil, err
+ }
+ field.SetConcrete(0, v)
+ default:
+ return nil, fmt.Errorf("unsupported field type: %s (%s)", f.Type, f.Name)
+ }
+ } else if f.Value == "#{now}" {
+ // Variable.
+ // TODO: make consistent with Grafana variables?
+ nowTimeFunc := c.nowTimeFunc
+ if nowTimeFunc == nil {
+ nowTimeFunc = time.Now
+ }
+ field.SetConcrete(0, nowTimeFunc())
+ }
+
+ labels := map[string]string{}
+ for _, label := range f.Labels {
+ if strings.HasPrefix(label.Value, "$") {
+ nodes, err := ajson.JSONPath(body, label.Value)
+ if err != nil {
+ return nil, err
+ }
+ if len(nodes) == 0 {
+ labels[label.Name] = ""
+ } else if len(nodes) == 1 {
+ value, err := nodes[0].Value()
+ if err != nil {
+ return nil, err
+ }
+ labels[label.Name] = fmt.Sprintf("%v", value)
+ } else {
+ return nil, errors.New("too many values for a label")
+ }
+ //x, err := jp.ParseString(label.Value[1:])
+ //if err != nil {
+ // return nil, err
+ //}
+ //value := x.Get(obj)
+ //if len(value) == 0 {
+ // labels[label.Name] = ""
+ //} else if len(value) == 1 {
+ // labels[label.Name] = fmt.Sprintf("%v", value[0])
+ //} else {
+ // return nil, errors.New("too many values for a label")
+ //}
+ } else if strings.HasPrefix(label.Value, "{") {
+ script := strings.Trim(label.Value, "{}")
+ var err error
+ initGojaOnce.Do(func() {
+ gojaRuntime, err = getRuntime(body)
+ })
+ if err != nil {
+ return nil, err
+ }
+ v, err := gojaRuntime.getString(script)
+ if err != nil {
+ return nil, err
+ }
+ labels[label.Name] = v
+ }
+ }
+ field.Labels = labels
+ fields = append(fields, field)
+ }
+
+ frame := data.NewFrame(vars.Path, fields...)
+ return []*ChannelFrame{
+ {Channel: "", Frame: frame},
+ }, nil
+}
diff --git a/pkg/services/live/pipeline/converter_json_exact_test.go b/pkg/services/live/pipeline/converter_json_exact_test.go
new file mode 100644
index 00000000000..6bb9845fd17
--- /dev/null
+++ b/pkg/services/live/pipeline/converter_json_exact_test.go
@@ -0,0 +1,68 @@
+package pipeline
+
+import (
+ "context"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/experimental"
+ "github.com/stretchr/testify/require"
+)
+
+func checkExactConversion(tb testing.TB, file string, fields []Field) *backend.DataResponse {
+ tb.Helper()
+ content := loadTestJson(tb, file)
+
+ converter := NewExactJsonConverter(ExactJsonConverterConfig{
+ Fields: fields,
+ })
+ converter.nowTimeFunc = func() time.Time {
+ return time.Date(2021, 01, 01, 12, 12, 12, 0, time.UTC)
+ }
+ channelFrames, err := converter.Convert(context.Background(), Vars{}, content)
+ require.NoError(tb, err)
+
+ dr := &backend.DataResponse{}
+ for _, cf := range channelFrames {
+ require.Empty(tb, cf.Channel)
+ dr.Frames = append(dr.Frames, cf.Frame)
+ }
+
+ err = experimental.CheckGoldenDataResponse(filepath.Join("testdata", file+".golden.txt"), dr, *update)
+ require.NoError(tb, err)
+ return dr
+}
+
+func TestExactJsonConverter_Convert(t *testing.T) {
+ checkExactConversion(t, "json_exact", []Field{
+ {
+ Name: "time",
+ Value: "#{now}",
+ Type: data.FieldTypeTime,
+ },
+ {
+ Name: "ax",
+ Value: "$.ax",
+ Type: data.FieldTypeNullableFloat64,
+ },
+ {
+ Name: "key1",
+ Value: "{x.map_with_floats.key1}",
+ Type: data.FieldTypeNullableFloat64,
+ Labels: []Label{
+ {
+ Name: "label1",
+ Value: "{x.map_with_floats.key2.toString()}",
+ },
+ {
+ Name: "label2",
+ Value: "$.map_with_floats.key2",
+ },
+ },
+ },
+ })
+}
diff --git a/pkg/services/live/pipeline/converter_json_frame.go b/pkg/services/live/pipeline/converter_json_frame.go
new file mode 100644
index 00000000000..d8a7426b720
--- /dev/null
+++ b/pkg/services/live/pipeline/converter_json_frame.go
@@ -0,0 +1,32 @@
+package pipeline
+
+import (
+ "context"
+ "encoding/json"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type JsonFrameConverterConfig struct{}
+
+// JsonFrameConverter decodes single data.Frame from JSON.
+type JsonFrameConverter struct {
+ config JsonFrameConverterConfig
+}
+
+func NewJsonFrameConverter(c JsonFrameConverterConfig) *JsonFrameConverter {
+ return &JsonFrameConverter{
+ config: c,
+ }
+}
+
+func (c *JsonFrameConverter) Convert(_ context.Context, _ Vars, body []byte) ([]*ChannelFrame, error) {
+ var frame data.Frame
+ err := json.Unmarshal(body, &frame)
+ if err != nil {
+ return nil, err
+ }
+ return []*ChannelFrame{
+ {Channel: "", Frame: &frame},
+ }, nil
+}
diff --git a/pkg/services/live/pipeline/devdata.go b/pkg/services/live/pipeline/devdata.go
new file mode 100644
index 00000000000..d80ae1a1187
--- /dev/null
+++ b/pkg/services/live/pipeline/devdata.go
@@ -0,0 +1,329 @@
+package pipeline
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "log"
+ "math/rand"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/centrifugal/centrifuge"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/grafana/grafana/pkg/services/live/managedstream"
+)
+
+type Data struct {
+ Value1 float64 `json:"value1"`
+ Value2 float64 `json:"value2"`
+ Value3 *float64 `json:"value3"`
+ Value4 float64 `json:"value4"`
+ Annotation string `json:"annotation"`
+ Array []float64 `json:"array"`
+ Map map[string]interface{} `json:"map"`
+ Host string `json:"host"`
+ Status string `json:"status"`
+}
+
+// TODO: temporary for development, remove.
+func postTestData() {
+ i := 0
+ for {
+ time.Sleep(1000 * time.Millisecond)
+ num1 := rand.Intn(10)
+ num2 := rand.Intn(10)
+ d := Data{
+ Value1: float64(num1),
+ Value2: float64(num2),
+ Value4: float64(i % 10),
+ Annotation: "odd",
+ Array: []float64{float64(rand.Intn(10)), float64(rand.Intn(10))},
+ Map: map[string]interface{}{
+ "red": 1,
+ "yellow": 4,
+ "green": 7,
+ },
+ Host: "macbook-local",
+ Status: "running",
+ }
+ if i%2 != 0 {
+ val := 4.0
+ d.Value3 = &val
+ }
+ if i%2 == 0 {
+ val := 3.0
+ d.Value3 = &val
+ d.Annotation = "even"
+ }
+ if i%10 == 0 {
+ d.Value3 = nil
+ }
+ jsonData, _ := json.Marshal(d)
+ log.Println(string(jsonData))
+
+ req, _ := http.NewRequest("POST", "http://localhost:3000/api/live/push/json/auto", bytes.NewReader(jsonData))
+ req.Header.Set("Authorization", "Bearer "+os.Getenv("GF_TOKEN"))
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ log.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ req, _ = http.NewRequest("POST", "http://localhost:3000/api/live/push/json/tip", bytes.NewReader(jsonData))
+ req.Header.Set("Authorization", "Bearer "+os.Getenv("GF_TOKEN"))
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ log.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ req, _ = http.NewRequest("POST", "http://localhost:3000/api/live/push/json/exact", bytes.NewReader(jsonData))
+ req.Header.Set("Authorization", "Bearer "+os.Getenv("GF_TOKEN"))
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ log.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ i++
+ }
+}
+
+type DevRuleBuilder struct {
+ Node *centrifuge.Node
+ ManagedStream *managedstream.Runner
+ FrameStorage *FrameStorage
+}
+
+func (f *DevRuleBuilder) BuildRules(_ context.Context, _ int64) ([]*LiveChannelRule, error) {
+ return []*LiveChannelRule{
+ {
+ Pattern: "plugin/testdata/random-20Hz-stream",
+ Converter: NewJsonFrameConverter(JsonFrameConverterConfig{}),
+ Outputter: NewMultipleOutput(
+ NewManagedStreamOutput(f.ManagedStream),
+ NewRedirectOutput(RedirectOutputConfig{
+ Channel: "stream/testdata/random-20Hz-stream",
+ }),
+ ),
+ },
+ {
+ Pattern: "stream/testdata/random-20Hz-stream",
+ Processor: NewKeepFieldsProcessor(KeepFieldsProcessorConfig{
+ FieldNames: []string{"Time", "Min", "Max"},
+ }),
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/influx/input",
+ Converter: NewAutoInfluxConverter(AutoInfluxConverterConfig{
+ FrameFormat: "labels_column",
+ }),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/influx/input/:rest",
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/influx/input/cpu",
+ // TODO: Would be fine to have KeepLabelsProcessor, but we need to know frame type
+ // since there are cases when labels attached to a field, and cases where labels
+ // set in a first frame column (in Influx converter). For example, this will allow
+ // to leave only "total-cpu" data while dropping individual CPUs.
+ Processor: NewKeepFieldsProcessor(KeepFieldsProcessorConfig{
+ FieldNames: []string{"labels", "time", "usage_user"},
+ }),
+ Outputter: NewMultipleOutput(
+ NewManagedStreamOutput(f.ManagedStream),
+ NewConditionalOutput(
+ NewNumberCompareCondition("usage_user", "gte", 50),
+ NewRedirectOutput(RedirectOutputConfig{
+ Channel: "stream/influx/input/cpu/spikes",
+ }),
+ ),
+ ),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/influx/input/cpu/spikes",
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/auto",
+ Converter: NewAutoJsonConverter(AutoJsonConverterConfig{}),
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/tip",
+ Converter: NewAutoJsonConverter(AutoJsonConverterConfig{
+ FieldTips: map[string]Field{
+ "value3": {
+ Name: "value3",
+ Type: data.FieldTypeNullableFloat64,
+ },
+ "value100": {
+ Name: "value100",
+ Type: data.FieldTypeNullableFloat64,
+ },
+ },
+ }),
+ Processor: NewDropFieldsProcessor(DropFieldsProcessorConfig{
+ FieldNames: []string{"value2"},
+ }),
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/exact",
+ Converter: NewExactJsonConverter(ExactJsonConverterConfig{
+ Fields: []Field{
+ {
+ Name: "time",
+ Type: data.FieldTypeTime,
+ Value: "#{now}",
+ },
+ {
+ Name: "value1",
+ Type: data.FieldTypeNullableFloat64,
+ Value: "$.value1",
+ },
+ {
+ Name: "value2",
+ Type: data.FieldTypeNullableFloat64,
+ Value: "$.value2",
+ },
+ {
+ Name: "value3",
+ Type: data.FieldTypeNullableFloat64,
+ Value: "$.value3",
+ Labels: []Label{
+ {
+ Name: "host",
+ Value: "$.host",
+ },
+ },
+ },
+ {
+ Name: "value4",
+ Type: data.FieldTypeNullableFloat64,
+ Value: "$.value4",
+ Config: &data.FieldConfig{
+ Thresholds: &data.ThresholdsConfig{
+ Mode: data.ThresholdsModeAbsolute,
+ Steps: []data.Threshold{
+ {
+ Value: 2,
+ State: "normal",
+ Color: "green",
+ },
+ {
+ Value: 6,
+ State: "warning",
+ Color: "orange",
+ },
+ {
+ Value: 8,
+ State: "critical",
+ Color: "red",
+ },
+ },
+ },
+ },
+ },
+ {
+ Name: "map.red",
+ Type: data.FieldTypeNullableFloat64,
+ Value: "$.map.red",
+ Labels: []Label{
+ {
+ Name: "host",
+ Value: "$.host",
+ },
+ {
+ Name: "host2",
+ Value: "$.host",
+ },
+ },
+ },
+ {
+ Name: "annotation",
+ Type: data.FieldTypeNullableString,
+ Value: "$.annotation",
+ },
+ {
+ Name: "running",
+ Type: data.FieldTypeNullableBool,
+ Value: "{x.status === 'running'}",
+ },
+ {
+ Name: "num_map_colors",
+ Type: data.FieldTypeNullableFloat64,
+ Value: "{Object.keys(x.map).length}",
+ },
+ },
+ }),
+ Outputter: NewMultipleOutput(
+ NewManagedStreamOutput(f.ManagedStream),
+ NewRemoteWriteOutput(RemoteWriteConfig{
+ Endpoint: os.Getenv("GF_LIVE_REMOTE_WRITE_ENDPOINT"),
+ User: os.Getenv("GF_LIVE_REMOTE_WRITE_USER"),
+ Password: os.Getenv("GF_LIVE_REMOTE_WRITE_PASSWORD"),
+ }),
+ NewChangeLogOutput(f.FrameStorage, ChangeLogOutputConfig{
+ FieldName: "value3",
+ Channel: "stream/json/exact/value3/changes",
+ }),
+ NewChangeLogOutput(f.FrameStorage, ChangeLogOutputConfig{
+ FieldName: "annotation",
+ Channel: "stream/json/exact/annotation/changes",
+ }),
+ NewConditionalOutput(
+ NewMultipleConditionChecker(
+ ConditionAll,
+ NewNumberCompareCondition("value1", "gte", 3.0),
+ NewNumberCompareCondition("value2", "gte", 3.0),
+ ),
+ NewRedirectOutput(RedirectOutputConfig{
+ Channel: "stream/json/exact/condition",
+ }),
+ ),
+ NewThresholdOutput(f.FrameStorage, ThresholdOutputConfig{
+ FieldName: "value4",
+ Channel: "stream/json/exact/value4/state",
+ }),
+ ),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/exact/value3/changes",
+ Outputter: NewMultipleOutput(
+ NewManagedStreamOutput(f.ManagedStream),
+ NewRemoteWriteOutput(RemoteWriteConfig{
+ Endpoint: os.Getenv("GF_LIVE_REMOTE_WRITE_ENDPOINT"),
+ User: os.Getenv("GF_LIVE_REMOTE_WRITE_USER"),
+ Password: os.Getenv("GF_LIVE_REMOTE_WRITE_PASSWORD"),
+ }),
+ ),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/exact/annotation/changes",
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/exact/condition",
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/json/exact/value4/state",
+ Outputter: NewManagedStreamOutput(f.ManagedStream),
+ },
+ }, nil
+}
diff --git a/pkg/services/live/pipeline/frame_storage.go b/pkg/services/live/pipeline/frame_storage.go
new file mode 100644
index 00000000000..517ebc7bf80
--- /dev/null
+++ b/pkg/services/live/pipeline/frame_storage.go
@@ -0,0 +1,37 @@
+package pipeline
+
+import (
+ "sync"
+
+ "github.com/grafana/grafana/pkg/services/live/orgchannel"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// FrameStorage keeps last channel frame in memory. Not usable in HA setup.
+type FrameStorage struct {
+ mu sync.RWMutex
+ frames map[string]*data.Frame
+}
+
+func NewFrameStorage() *FrameStorage {
+ return &FrameStorage{
+ frames: map[string]*data.Frame{},
+ }
+}
+
+func (s *FrameStorage) Set(orgID int64, channel string, frame *data.Frame) error {
+ key := orgchannel.PrependOrgID(orgID, channel)
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.frames[key] = frame
+ return nil
+}
+
+func (s *FrameStorage) Get(orgID int64, channel string) (*data.Frame, bool, error) {
+ key := orgchannel.PrependOrgID(orgID, channel)
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ f, ok := s.frames[key]
+ return f, ok, nil
+}
diff --git a/pkg/services/live/pipeline/goja_expression.go b/pkg/services/live/pipeline/goja_expression.go
new file mode 100644
index 00000000000..0ce1b126de0
--- /dev/null
+++ b/pkg/services/live/pipeline/goja_expression.go
@@ -0,0 +1,97 @@
+package pipeline
+
+import (
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/dop251/goja"
+ "github.com/dop251/goja/parser"
+)
+
+func getRuntime(payload []byte) (*gojaRuntime, error) {
+ vm := goja.New()
+ vm.SetMaxCallStackSize(64)
+ vm.SetParserOptions(parser.WithDisableSourceMaps)
+ r := &gojaRuntime{vm}
+ err := r.init(payload)
+ if err != nil {
+ return nil, err
+ }
+ return r, nil
+}
+
+type gojaRuntime struct {
+ vm *goja.Runtime
+}
+
+// Parse JSON once.
+func (r *gojaRuntime) init(payload []byte) error {
+ err := r.vm.Set("__body", string(payload))
+ if err != nil {
+ return err
+ }
+ _, err = r.runString(`var x = JSON.parse(__body)`)
+ return err
+}
+
+func (r *gojaRuntime) runString(script string) (goja.Value, error) {
+ doneCh := make(chan struct{})
+ go func() {
+ select {
+ case <-doneCh:
+ return
+ case <-time.After(100 * time.Millisecond):
+ // Some ideas to prevent misuse of scripts:
+ // * parse/validate scripts on save
+ // * block scripts after several timeouts in a row
+ // * block scripts on malformed returned error
+ // * limit total quota of time for scripts
+ // * maybe allow only one statement, reject scripts with cycles and functions.
+ r.vm.Interrupt(errors.New("timeout"))
+ }
+ }()
+ defer close(doneCh)
+ return r.vm.RunString(script)
+}
+
+func (r *gojaRuntime) getBool(script string) (bool, error) {
+ v, err := r.runString(script)
+ if err != nil {
+ return false, err
+ }
+ num, ok := v.Export().(bool)
+ if !ok {
+ return false, errors.New("unexpected return value")
+ }
+ return num, nil
+}
+
+func (r *gojaRuntime) getString(script string) (string, error) {
+ v, err := r.runString(script)
+ if err != nil {
+ return "", err
+ }
+ exportedVal := v.Export()
+ stringVal, ok := exportedVal.(string)
+ if !ok {
+ return "", fmt.Errorf("unexpected return value: %v (%T), script: %s", exportedVal, exportedVal, script)
+ }
+ return stringVal, nil
+}
+
+func (r *gojaRuntime) getFloat64(script string) (float64, error) {
+ v, err := r.runString(script)
+ if err != nil {
+ return 0, err
+ }
+ exported := v.Export()
+ switch v := exported.(type) {
+ case float64:
+ return v, nil
+ case int64:
+ return float64(v), nil
+ default:
+ return 0, fmt.Errorf("unexpected return value: %T", exported)
+ }
+}
diff --git a/pkg/services/live/pipeline/goja_expression_test.go b/pkg/services/live/pipeline/goja_expression_test.go
new file mode 100644
index 00000000000..1f7cc518a91
--- /dev/null
+++ b/pkg/services/live/pipeline/goja_expression_test.go
@@ -0,0 +1,55 @@
+package pipeline
+
+import (
+ "testing"
+
+ "github.com/dop251/goja"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGojaGetBool(t *testing.T) {
+ r, err := getRuntime([]byte(`{"ax": true}`))
+ require.NoError(t, err)
+ val, err := r.getBool("x.ax")
+ require.NoError(t, err)
+ require.True(t, val)
+}
+
+func TestGojaGetFloat64(t *testing.T) {
+ r, err := getRuntime([]byte(`{"ax": 3}`))
+ require.NoError(t, err)
+ val, err := r.getFloat64("x.ax")
+ require.NoError(t, err)
+ require.Equal(t, 3.0, val)
+}
+
+func TestGojaGetString(t *testing.T) {
+ r, err := getRuntime([]byte(`{"ax": "test"}`))
+ require.NoError(t, err)
+ val, err := r.getString("x.ax")
+ require.NoError(t, err)
+ require.Equal(t, "test", val)
+}
+
+func TestGojaInvalidReturnValue(t *testing.T) {
+ r, err := getRuntime([]byte(`{"ax": "test"}`))
+ require.NoError(t, err)
+ _, err = r.getBool("x.ax")
+ require.Error(t, err)
+}
+
+func TestGojaIInterrupt(t *testing.T) {
+ r, err := getRuntime([]byte(`{}`))
+ require.NoError(t, err)
+ _, err = r.getBool("while (true) {}")
+ var interrupted *goja.InterruptedError
+ require.ErrorAs(t, err, &interrupted)
+}
+
+func TestGojaIMaxStack(t *testing.T) {
+ r, err := getRuntime([]byte(`{}`))
+ require.NoError(t, err)
+ _, err = r.getBool("function test() {test()}; test();")
+ // TODO: strange error returned here, need to investigate what is it.
+ require.Error(t, err)
+}
diff --git a/pkg/services/live/pipeline/json_to_frame.go b/pkg/services/live/pipeline/json_to_frame.go
new file mode 100644
index 00000000000..6796b63d504
--- /dev/null
+++ b/pkg/services/live/pipeline/json_to_frame.go
@@ -0,0 +1,135 @@
+package pipeline
+
+import (
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ jsoniter "github.com/json-iterator/go"
+)
+
+type doc struct {
+ path []string
+ iterator *jsoniter.Iterator
+ fields []*data.Field
+ fieldNames map[string]struct{}
+ fieldTips map[string]Field
+}
+
+func (d *doc) next() error {
+ switch d.iterator.WhatIsNext() {
+ case jsoniter.StringValue:
+ d.addString(d.iterator.ReadString())
+ case jsoniter.NumberValue:
+ d.addNumber(d.iterator.ReadFloat64())
+ case jsoniter.BoolValue:
+ d.addBool(d.iterator.ReadBool())
+ case jsoniter.NilValue:
+ d.addNil()
+ d.iterator.ReadNil()
+ case jsoniter.ArrayValue:
+ index := 0
+ size := len(d.path)
+ for d.iterator.ReadArray() {
+ d.path = append(d.path, fmt.Sprintf("[%d]", index))
+ err := d.next()
+ if err != nil {
+ return err
+ }
+ d.path = d.path[:size]
+ index++
+ }
+ case jsoniter.ObjectValue:
+ size := len(d.path)
+ for fname := d.iterator.ReadObject(); fname != ""; fname = d.iterator.ReadObject() {
+ if size > 0 {
+ d.path = append(d.path, ".")
+ }
+ d.path = append(d.path, fname)
+ err := d.next()
+ if err != nil {
+ return err
+ }
+ d.path = d.path[:size]
+ }
+ case jsoniter.InvalidValue:
+ return fmt.Errorf("invalid value")
+ }
+ return nil
+}
+
+func (d *doc) key() string {
+ return strings.Join(d.path, "")
+}
+
+func (d *doc) addString(v string) {
+ f := data.NewFieldFromFieldType(data.FieldTypeNullableString, 1)
+ f.Name = d.key()
+ f.SetConcrete(0, v)
+ d.fields = append(d.fields, f)
+ d.fieldNames[d.key()] = struct{}{}
+}
+
+func (d *doc) addNumber(v float64) {
+ f := data.NewFieldFromFieldType(data.FieldTypeNullableFloat64, 1)
+ f.Name = d.key()
+ f.SetConcrete(0, v)
+ d.fields = append(d.fields, f)
+ d.fieldNames[d.key()] = struct{}{}
+}
+
+func (d *doc) addBool(v bool) {
+ f := data.NewFieldFromFieldType(data.FieldTypeNullableBool, 1)
+ f.Name = d.key()
+ f.SetConcrete(0, v)
+ d.fields = append(d.fields, f)
+ d.fieldNames[d.key()] = struct{}{}
+}
+
+func (d *doc) addNil() {
+ if tip, ok := d.fieldTips[d.key()]; ok {
+ f := data.NewFieldFromFieldType(tip.Type, 1)
+ f.Name = d.key()
+ f.Set(0, nil)
+ d.fields = append(d.fields, f)
+ d.fieldNames[d.key()] = struct{}{}
+ } else {
+ logger.Warn("Skip nil field", "key", d.key())
+ }
+}
+
+func jsonDocToFrame(name string, body []byte, fields map[string]Field, nowTimeFunc func() time.Time) (*data.Frame, error) {
+ d := doc{
+ iterator: jsoniter.ParseBytes(jsoniter.ConfigDefault, body),
+ path: make([]string, 0),
+ fieldTips: fields,
+ fieldNames: map[string]struct{}{},
+ }
+
+ f := data.NewFieldFromFieldType(data.FieldTypeTime, 1)
+ f.Set(0, nowTimeFunc())
+ d.fields = append(d.fields, f)
+
+ err := d.next()
+ if err != nil {
+ return nil, err
+ }
+
+ if len(d.fields) < 2 {
+ return nil, fmt.Errorf("no fields found")
+ }
+
+ for name, tip := range fields {
+ if _, ok := d.fieldNames[name]; ok {
+ continue
+ }
+ f := data.NewFieldFromFieldType(tip.Type, 1)
+ f.Name = name
+ f.Set(0, nil)
+ f.Config = tip.Config
+ d.fields = append(d.fields, f)
+ }
+
+ return data.NewFrame(name, d.fields...), nil
+}
diff --git a/pkg/services/live/pipeline/logger.go b/pkg/services/live/pipeline/logger.go
new file mode 100644
index 00000000000..0b85054d47e
--- /dev/null
+++ b/pkg/services/live/pipeline/logger.go
@@ -0,0 +1,7 @@
+package pipeline
+
+import "github.com/grafana/grafana/pkg/infra/log"
+
+var (
+ logger = log.New("live.pipeline")
+)
diff --git a/pkg/services/live/pipeline/output_changelog.go b/pkg/services/live/pipeline/output_changelog.go
new file mode 100644
index 00000000000..91072a7f586
--- /dev/null
+++ b/pkg/services/live/pipeline/output_changelog.go
@@ -0,0 +1,92 @@
+package pipeline
+
+import (
+ "context"
+ "reflect"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type ChangeLogOutputConfig struct {
+ FieldName string `json:"fieldName"`
+ Channel string `json:"channel"`
+}
+
+// ChangeLogOutput can monitor value changes of the specified field and output
+// special change frame to the configured channel.
+type ChangeLogOutput struct {
+ frameStorage FrameGetSetter
+ config ChangeLogOutputConfig
+}
+
+func NewChangeLogOutput(frameStorage FrameGetSetter, config ChangeLogOutputConfig) *ChangeLogOutput {
+ return &ChangeLogOutput{frameStorage: frameStorage, config: config}
+}
+
+func (l ChangeLogOutput) Output(_ context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ previousFrame, previousFrameOK, err := l.frameStorage.Get(vars.OrgID, l.config.Channel)
+ if err != nil {
+ return nil, err
+ }
+
+ fieldName := l.config.FieldName
+
+ previousFrameFieldIndex := -1
+ if previousFrameOK {
+ for i, f := range previousFrame.Fields {
+ if f.Name == fieldName {
+ previousFrameFieldIndex = i
+ }
+ }
+ }
+
+ currentFrameFieldIndex := -1
+ for i, f := range frame.Fields {
+ if f.Name == fieldName {
+ currentFrameFieldIndex = i
+ }
+ }
+
+ var previousValue interface{}
+ if previousFrameFieldIndex >= 0 {
+ // Take last value for the field.
+ previousValue = previousFrame.Fields[previousFrameFieldIndex].At(previousFrame.Fields[previousFrameFieldIndex].Len() - 1)
+ }
+
+ fTime := data.NewFieldFromFieldType(data.FieldTypeTime, 0)
+ fTime.Name = "time"
+ f1 := data.NewFieldFromFieldType(frame.Fields[currentFrameFieldIndex].Type(), 0)
+ f1.Name = "old"
+ f2 := data.NewFieldFromFieldType(frame.Fields[currentFrameFieldIndex].Type(), 0)
+ f2.Name = "new"
+
+ if currentFrameFieldIndex >= 0 {
+ for i := 0; i < frame.Fields[currentFrameFieldIndex].Len(); i++ {
+ currentValue := frame.Fields[currentFrameFieldIndex].At(i)
+ if !reflect.DeepEqual(
+ previousValue,
+ currentValue,
+ ) {
+ fTime.Append(time.Now())
+ f1.Append(previousValue)
+ f2.Append(currentValue)
+ previousValue = currentValue
+ }
+ }
+ }
+
+ if fTime.Len() > 0 {
+ changeFrame := data.NewFrame("change", fTime, f1, f2)
+ err := l.frameStorage.Set(vars.OrgID, l.config.Channel, frame)
+ if err != nil {
+ return nil, err
+ }
+ return []*ChannelFrame{{
+ Channel: l.config.Channel,
+ Frame: changeFrame,
+ }}, nil
+ }
+
+ return nil, l.frameStorage.Set(vars.OrgID, l.config.Channel, frame)
+}
diff --git a/pkg/services/live/pipeline/output_changelog_test.go b/pkg/services/live/pipeline/output_changelog_test.go
new file mode 100644
index 00000000000..13c88dac948
--- /dev/null
+++ b/pkg/services/live/pipeline/output_changelog_test.go
@@ -0,0 +1,90 @@
+package pipeline
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/golang/mock/gomock"
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/stretchr/testify/require"
+)
+
+func TestChangeLogOutput_NoPreviousFrame_SingleRow(t *testing.T) {
+ mockCtrl := gomock.NewController(t)
+ defer mockCtrl.Finish()
+
+ mockStorage := NewMockFrameGetSetter(mockCtrl)
+
+ mockStorage.EXPECT().Get(gomock.Any(), gomock.Any()).DoAndReturn(func(orgID int64, channel string) (*data.Frame, bool, error) {
+ return nil, false, nil
+ })
+
+ mockStorage.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Times(1)
+
+ outputter := NewChangeLogOutput(mockStorage, ChangeLogOutputConfig{
+ FieldName: "test",
+ Channel: "stream/test/no_previous_frame",
+ })
+
+ f1 := data.NewField("time", nil, make([]time.Time, 1))
+ f1.Set(0, time.Now())
+
+ f2 := data.NewField("test", nil, make([]*float64, 1))
+ f2.SetConcrete(0, 20.0)
+
+ frame := data.NewFrame("test", f1, f2)
+
+ channelFrames, err := outputter.Output(context.Background(), OutputVars{}, frame)
+ require.NoError(t, err)
+
+ require.Len(t, channelFrames, 1)
+ changeFrame := channelFrames[0].Frame
+ require.Len(t, changeFrame.Fields, 3)
+ var x *float64
+ var y = 20.0
+ require.Equal(t, x, changeFrame.Fields[1].At(0).(*float64))
+ require.Equal(t, &y, changeFrame.Fields[2].At(0))
+}
+
+func TestChangeLogOutput_NoPreviousFrame_MultipleRows(t *testing.T) {
+ mockCtrl := gomock.NewController(t)
+ defer mockCtrl.Finish()
+
+ mockStorage := NewMockFrameGetSetter(mockCtrl)
+
+ mockStorage.EXPECT().Get(gomock.Any(), gomock.Any()).DoAndReturn(func(orgID int64, channel string) (*data.Frame, bool, error) {
+ return nil, false, nil
+ }).Times(1)
+
+ mockStorage.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Times(1)
+
+ outputter := NewChangeLogOutput(mockStorage, ChangeLogOutputConfig{
+ FieldName: "test",
+ Channel: "stream/test/no_previous_frame",
+ })
+
+ f1 := data.NewField("time", nil, make([]time.Time, 2))
+ f1.Set(0, time.Now())
+ f1.Set(1, time.Now())
+
+ f2 := data.NewField("test", nil, make([]*float64, 2))
+ f2.SetConcrete(0, 5.0)
+ f2.SetConcrete(1, 20.0)
+
+ frame := data.NewFrame("test", f1, f2)
+
+ channelFrames, err := outputter.Output(context.Background(), OutputVars{}, frame)
+ require.NoError(t, err)
+ require.Len(t, channelFrames, 1)
+ changeFrame := channelFrames[0].Frame
+ require.Len(t, changeFrame.Fields, 3)
+ var x *float64
+ var y = 5.0
+ require.Equal(t, x, changeFrame.Fields[1].At(0).(*float64))
+ require.Equal(t, &y, changeFrame.Fields[2].At(0))
+ var z = 5.0
+ var v = 20.0
+ require.Equal(t, &z, changeFrame.Fields[1].At(1).(*float64))
+ require.Equal(t, &v, changeFrame.Fields[2].At(1))
+}
diff --git a/pkg/services/live/pipeline/output_conditional.go b/pkg/services/live/pipeline/output_conditional.go
new file mode 100644
index 00000000000..e47960c74e1
--- /dev/null
+++ b/pkg/services/live/pipeline/output_conditional.go
@@ -0,0 +1,27 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type ConditionalOutput struct {
+ Condition ConditionChecker
+ Outputter Outputter
+}
+
+func NewConditionalOutput(condition ConditionChecker, outputter Outputter) *ConditionalOutput {
+ return &ConditionalOutput{Condition: condition, Outputter: outputter}
+}
+
+func (l ConditionalOutput) Output(ctx context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ ok, err := l.Condition.CheckCondition(ctx, frame)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, nil
+ }
+ return l.Outputter.Output(ctx, vars, frame)
+}
diff --git a/pkg/services/live/pipeline/output_local_subscribers.go b/pkg/services/live/pipeline/output_local_subscribers.go
new file mode 100644
index 00000000000..5a8a2931536
--- /dev/null
+++ b/pkg/services/live/pipeline/output_local_subscribers.go
@@ -0,0 +1,38 @@
+package pipeline
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "github.com/grafana/grafana/pkg/services/live/orgchannel"
+
+ "github.com/centrifugal/centrifuge"
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type LocalSubscribersOutput struct {
+ // TODO: refactor to depend on interface (avoid Centrifuge dependency here).
+ node *centrifuge.Node
+}
+
+func NewLocalSubscribersOutput(node *centrifuge.Node) *LocalSubscribersOutput {
+ return &LocalSubscribersOutput{node: node}
+}
+
+func (l *LocalSubscribersOutput) Output(_ context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ channelID := vars.Channel
+ channel := orgchannel.PrependOrgID(vars.OrgID, channelID)
+ frameJSON, err := json.Marshal(frame)
+ if err != nil {
+ return nil, err
+ }
+ pub := ¢rifuge.Publication{
+ Data: frameJSON,
+ }
+ err = l.node.Hub().BroadcastPublication(channel, pub, centrifuge.StreamPosition{})
+ if err != nil {
+ return nil, fmt.Errorf("error publishing %s: %w", string(frameJSON), err)
+ }
+ return nil, nil
+}
diff --git a/pkg/services/live/pipeline/output_managed_stream.go b/pkg/services/live/pipeline/output_managed_stream.go
new file mode 100644
index 00000000000..bd225d704a4
--- /dev/null
+++ b/pkg/services/live/pipeline/output_managed_stream.go
@@ -0,0 +1,26 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana/pkg/services/live/managedstream"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type ManagedStreamOutput struct {
+ managedStream *managedstream.Runner
+}
+
+func NewManagedStreamOutput(managedStream *managedstream.Runner) *ManagedStreamOutput {
+ return &ManagedStreamOutput{managedStream: managedStream}
+}
+
+func (l *ManagedStreamOutput) Output(_ context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ stream, err := l.managedStream.GetOrCreateStream(vars.OrgID, vars.Scope, vars.Namespace)
+ if err != nil {
+ logger.Error("Error getting stream", "error", err)
+ return nil, err
+ }
+ return nil, stream.Push(vars.Path, frame)
+}
diff --git a/pkg/services/live/pipeline/output_multiple.go b/pkg/services/live/pipeline/output_multiple.go
new file mode 100644
index 00000000000..d763bf259ec
--- /dev/null
+++ b/pkg/services/live/pipeline/output_multiple.go
@@ -0,0 +1,30 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// MultipleOutput can combine several Outputter and
+// execute them sequentially.
+type MultipleOutput struct {
+ Outputters []Outputter
+}
+
+func (m MultipleOutput) Output(ctx context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ var frames []*ChannelFrame
+ for _, out := range m.Outputters {
+ f, err := out.Output(ctx, vars, frame)
+ if err != nil {
+ logger.Error("Error outputting frame", "error", err)
+ return nil, err
+ }
+ frames = append(frames, f...)
+ }
+ return frames, nil
+}
+
+func NewMultipleOutput(outputters ...Outputter) *MultipleOutput {
+ return &MultipleOutput{Outputters: outputters}
+}
diff --git a/pkg/services/live/pipeline/output_redirect.go b/pkg/services/live/pipeline/output_redirect.go
new file mode 100644
index 00000000000..d562af77cf3
--- /dev/null
+++ b/pkg/services/live/pipeline/output_redirect.go
@@ -0,0 +1,33 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// RedirectOutputConfig ...
+type RedirectOutputConfig struct {
+ Channel string `json:"channel"`
+}
+
+// RedirectOutput passes processing control to the rule defined
+// for a configured channel.
+type RedirectOutput struct {
+ config RedirectOutputConfig
+}
+
+func NewRedirectOutput(config RedirectOutputConfig) *RedirectOutput {
+ return &RedirectOutput{config: config}
+}
+
+func (l *RedirectOutput) Output(_ context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ if vars.Channel == l.config.Channel {
+ return nil, fmt.Errorf("redirect to the same channel: %s", l.config.Channel)
+ }
+ return []*ChannelFrame{{
+ Channel: l.config.Channel,
+ Frame: frame,
+ }}, nil
+}
diff --git a/pkg/services/live/pipeline/output_remote_write.go b/pkg/services/live/pipeline/output_remote_write.go
new file mode 100644
index 00000000000..8d6eb10457f
--- /dev/null
+++ b/pkg/services/live/pipeline/output_remote_write.go
@@ -0,0 +1,72 @@
+package pipeline
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "net/http"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/grafana/grafana/pkg/services/live/remotewrite"
+)
+
+type RemoteWriteConfig struct {
+ // Endpoint to send streaming frames to.
+ Endpoint string `json:"endpoint"`
+ // User is a user for remote write request.
+ User string `json:"user"`
+ // Password for remote write endpoint.
+ Password string `json:"password"`
+}
+
+type RemoteWriteOutput struct {
+ config RemoteWriteConfig
+ httpClient *http.Client
+}
+
+func NewRemoteWriteOutput(config RemoteWriteConfig) *RemoteWriteOutput {
+ return &RemoteWriteOutput{
+ config: config,
+ httpClient: &http.Client{Timeout: 2 * time.Second},
+ }
+}
+
+func (r RemoteWriteOutput) Output(_ context.Context, _ OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ if r.config.Endpoint == "" {
+ logger.Debug("Skip sending to remote write: no url")
+ return nil, nil
+ }
+
+ // Use remote write for a stream.
+ remoteWriteData, err := remotewrite.SerializeLabelsColumn(frame)
+ if err != nil {
+ logger.Error("Error serializing to remote write format", "error", err)
+ return nil, err
+ }
+
+ logger.Debug("Sending to remote write endpoint", "url", r.config.Endpoint, "bodyLength", len(remoteWriteData))
+ req, err := http.NewRequest(http.MethodPost, r.config.Endpoint, bytes.NewReader(remoteWriteData))
+ if err != nil {
+ logger.Error("Error constructing remote write request", "error", err)
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ req.Header.Set("Content-Encoding", "snappy")
+ req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+ req.SetBasicAuth(r.config.User, r.config.Password)
+
+ started := time.Now()
+ resp, err := r.httpClient.Do(req)
+ if err != nil {
+ logger.Error("Error sending remote write request", "error", err)
+ return nil, err
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ logger.Error("Unexpected response code from remote write endpoint", "code", resp.StatusCode)
+ return nil, errors.New("unexpected response code from remote write endpoint")
+ }
+ logger.Debug("Successfully sent to remote write endpoint", "url", r.config.Endpoint, "elapsed", time.Since(started))
+ return nil, nil
+}
diff --git a/pkg/services/live/pipeline/output_threshold.go b/pkg/services/live/pipeline/output_threshold.go
new file mode 100644
index 00000000000..bda29b2b5fa
--- /dev/null
+++ b/pkg/services/live/pipeline/output_threshold.go
@@ -0,0 +1,150 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type ThresholdOutputConfig struct {
+ FieldName string `json:"fieldName"`
+ Channel string `json:"channel"`
+}
+
+//go:generate mockgen -destination=output_threshold_mock.go -package=pipeline github.com/grafana/grafana/pkg/services/live/pipeline FrameGetSetter
+
+type FrameGetSetter interface {
+ Get(orgID int64, channel string) (*data.Frame, bool, error)
+ Set(orgID int64, channel string, frame *data.Frame) error
+}
+
+// ThresholdOutput can monitor threshold transitions of the specified field and output
+// special state frame to the configured channel.
+type ThresholdOutput struct {
+ frameStorage FrameGetSetter
+ config ThresholdOutputConfig
+}
+
+func NewThresholdOutput(frameStorage FrameGetSetter, config ThresholdOutputConfig) *ThresholdOutput {
+ return &ThresholdOutput{frameStorage: frameStorage, config: config}
+}
+
+func (l *ThresholdOutput) Output(_ context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ if frame == nil {
+ return nil, nil
+ }
+ previousFrame, previousFrameOk, err := l.frameStorage.Get(vars.OrgID, l.config.Channel)
+ if err != nil {
+ return nil, err
+ }
+ fieldName := l.config.FieldName
+
+ currentFrameFieldIndex := -1
+ for i, f := range frame.Fields {
+ if f.Name == fieldName {
+ currentFrameFieldIndex = i
+ }
+ }
+ if currentFrameFieldIndex < 0 {
+ return nil, nil
+ }
+ if frame.Fields[currentFrameFieldIndex].Config == nil {
+ return nil, nil
+ }
+ if frame.Fields[currentFrameFieldIndex].Config.Thresholds == nil {
+ return nil, nil
+ }
+
+ mode := frame.Fields[currentFrameFieldIndex].Config.Thresholds.Mode
+ if mode != data.ThresholdsModeAbsolute {
+ return nil, fmt.Errorf("unsupported threshold mode: %s", mode)
+ }
+
+ if len(frame.Fields[currentFrameFieldIndex].Config.Thresholds.Steps) == 0 {
+ return nil, nil
+ }
+
+ previousFrameFieldIndex := -1
+ if previousFrameOk {
+ for i, f := range previousFrame.Fields {
+ if f.Name == fieldName {
+ previousFrameFieldIndex = i
+ }
+ }
+ }
+
+ var previousState *string
+ if previousFrameOk && previousFrameFieldIndex >= 0 {
+ var previousThreshold data.Threshold
+ value, ok := previousFrame.Fields[previousFrameFieldIndex].At(previousFrame.Fields[0].Len() - 1).(*float64)
+ if !ok {
+ return nil, nil
+ }
+ if value == nil {
+ // TODO: what should we do here?
+ return nil, nil
+ }
+ emptyState := ""
+ previousState = &emptyState
+ for _, threshold := range frame.Fields[currentFrameFieldIndex].Config.Thresholds.Steps {
+ if *value >= float64(threshold.Value) {
+ previousThreshold = threshold
+ previousState = &previousThreshold.State
+ continue
+ }
+ break
+ }
+ }
+
+ fTime := data.NewFieldFromFieldType(data.FieldTypeTime, 0)
+ fTime.Name = "time"
+ f1 := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0)
+ f1.Name = "value"
+ f2 := data.NewFieldFromFieldType(data.FieldTypeString, 0)
+ f2.Name = "state"
+ f3 := data.NewFieldFromFieldType(data.FieldTypeString, 0)
+ f3.Name = "color"
+
+ for i := 0; i < frame.Fields[currentFrameFieldIndex].Len(); i++ {
+ // TODO: support other numeric types.
+ value, ok := frame.Fields[currentFrameFieldIndex].At(i).(*float64)
+ if !ok {
+ return nil, nil
+ }
+ if value == nil {
+ // TODO: what should we do here?
+ return nil, nil
+ }
+ var currentThreshold data.Threshold
+ for _, threshold := range frame.Fields[currentFrameFieldIndex].Config.Thresholds.Steps {
+ if *value >= float64(threshold.Value) {
+ currentThreshold = threshold
+ continue
+ }
+ break
+ }
+ if previousState == nil || currentThreshold.State != *previousState {
+ fTime.Append(time.Now())
+ f1.Append(*value)
+ f2.Append(currentThreshold.State)
+ f3.Append(currentThreshold.Color)
+ previousState = ¤tThreshold.State
+ }
+ }
+
+ if fTime.Len() > 0 {
+ stateFrame := data.NewFrame("state", fTime, f1, f2, f3)
+ err := l.frameStorage.Set(vars.OrgID, l.config.Channel, frame)
+ if err != nil {
+ return nil, err
+ }
+ return []*ChannelFrame{{
+ Channel: l.config.Channel,
+ Frame: stateFrame,
+ }}, nil
+ }
+
+ return nil, l.frameStorage.Set(vars.OrgID, l.config.Channel, frame)
+}
diff --git a/pkg/services/live/pipeline/output_threshold_mock.go b/pkg/services/live/pipeline/output_threshold_mock.go
new file mode 100644
index 00000000000..42d27f7f440
--- /dev/null
+++ b/pkg/services/live/pipeline/output_threshold_mock.go
@@ -0,0 +1,65 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/grafana/grafana/pkg/services/live/pipeline (interfaces: FrameGetSetter)
+
+// Package pipeline is a generated GoMock package.
+package pipeline
+
+import (
+ reflect "reflect"
+
+ gomock "github.com/golang/mock/gomock"
+ data "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// MockFrameGetSetter is a mock of FrameGetSetter interface.
+type MockFrameGetSetter struct {
+ ctrl *gomock.Controller
+ recorder *MockFrameGetSetterMockRecorder
+}
+
+// MockFrameGetSetterMockRecorder is the mock recorder for MockFrameGetSetter.
+type MockFrameGetSetterMockRecorder struct {
+ mock *MockFrameGetSetter
+}
+
+// NewMockFrameGetSetter creates a new mock instance.
+func NewMockFrameGetSetter(ctrl *gomock.Controller) *MockFrameGetSetter {
+ mock := &MockFrameGetSetter{ctrl: ctrl}
+ mock.recorder = &MockFrameGetSetterMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockFrameGetSetter) EXPECT() *MockFrameGetSetterMockRecorder {
+ return m.recorder
+}
+
+// Get mocks base method.
+func (m *MockFrameGetSetter) Get(arg0 int64, arg1 string) (*data.Frame, bool, error) {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "Get", arg0, arg1)
+ ret0, _ := ret[0].(*data.Frame)
+ ret1, _ := ret[1].(bool)
+ ret2, _ := ret[2].(error)
+ return ret0, ret1, ret2
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockFrameGetSetterMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockFrameGetSetter)(nil).Get), arg0, arg1)
+}
+
+// Set mocks base method.
+func (m *MockFrameGetSetter) Set(arg0 int64, arg1 string, arg2 *data.Frame) error {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "Set", arg0, arg1, arg2)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// Set indicates an expected call of Set.
+func (mr *MockFrameGetSetterMockRecorder) Set(arg0, arg1, arg2 interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockFrameGetSetter)(nil).Set), arg0, arg1, arg2)
+}
diff --git a/pkg/services/live/pipeline/output_threshold_test.go b/pkg/services/live/pipeline/output_threshold_test.go
new file mode 100644
index 00000000000..6f063ea41ba
--- /dev/null
+++ b/pkg/services/live/pipeline/output_threshold_test.go
@@ -0,0 +1,254 @@
+package pipeline
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/golang/mock/gomock"
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/stretchr/testify/require"
+)
+
+func TestThresholdOutput_Output(t *testing.T) {
+ type fields struct {
+ frameStorage FrameGetSetter
+ config ThresholdOutputConfig
+ }
+ type args struct {
+ in0 context.Context
+ vars OutputVars
+ frame *data.Frame
+ }
+ tests := []struct {
+ name string
+ fields fields
+ args args
+ wantErr bool
+ }{
+ {
+ name: "nil_input_frame",
+ fields: fields{
+ frameStorage: nil,
+ config: ThresholdOutputConfig{
+ Channel: "test",
+ },
+ },
+ args: args{in0: context.Background(), vars: OutputVars{}, frame: nil},
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ l := &ThresholdOutput{
+ frameStorage: tt.fields.frameStorage,
+ config: tt.fields.config,
+ }
+ if _, err := l.Output(tt.args.in0, tt.args.vars, tt.args.frame); (err != nil) != tt.wantErr {
+ t.Errorf("Output() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestThresholdOutput_NoPreviousFrame_SingleRow(t *testing.T) {
+ mockCtrl := gomock.NewController(t)
+ defer mockCtrl.Finish()
+
+ mockStorage := NewMockFrameGetSetter(mockCtrl)
+
+ mockStorage.EXPECT().Get(gomock.Any(), gomock.Any()).DoAndReturn(func(orgID int64, channel string) (*data.Frame, bool, error) {
+ return nil, false, nil
+ })
+
+ mockStorage.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Times(1)
+
+ outputter := NewThresholdOutput(mockStorage, ThresholdOutputConfig{
+ FieldName: "test",
+ Channel: "stream/test/no_previous_frame",
+ })
+
+ f1 := data.NewField("time", nil, make([]time.Time, 1))
+ f1.Set(0, time.Now())
+
+ f2 := data.NewField("test", nil, make([]*float64, 1))
+ f2.SetConcrete(0, 20.0)
+ f2.Config = &data.FieldConfig{
+ Thresholds: &data.ThresholdsConfig{
+ Mode: data.ThresholdsModeAbsolute,
+ Steps: []data.Threshold{
+ {
+ Value: 10,
+ State: "normal",
+ Color: "green",
+ },
+ },
+ },
+ }
+
+ frame := data.NewFrame("test", f1, f2)
+
+ channelFrames, err := outputter.Output(context.Background(), OutputVars{}, frame)
+ require.NoError(t, err)
+
+ require.Len(t, channelFrames, 1)
+ stateFrame := channelFrames[0].Frame
+ require.Len(t, stateFrame.Fields, 4)
+ require.Equal(t, 20.0, stateFrame.Fields[1].At(0))
+ require.Equal(t, "normal", stateFrame.Fields[2].At(0))
+ require.Equal(t, "green", stateFrame.Fields[3].At(0))
+}
+
+func TestThresholdOutput_NoPreviousFrame_MultipleRows(t *testing.T) {
+ mockCtrl := gomock.NewController(t)
+ defer mockCtrl.Finish()
+
+ mockStorage := NewMockFrameGetSetter(mockCtrl)
+
+ mockStorage.EXPECT().Get(gomock.Any(), gomock.Any()).DoAndReturn(func(orgID int64, channel string) (*data.Frame, bool, error) {
+ return nil, false, nil
+ }).Times(1)
+
+ mockStorage.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Times(1)
+
+ outputter := NewThresholdOutput(mockStorage, ThresholdOutputConfig{
+ FieldName: "test",
+ Channel: "stream/test/no_previous_frame",
+ })
+
+ f1 := data.NewField("time", nil, make([]time.Time, 2))
+ f1.Set(0, time.Now())
+ f1.Set(1, time.Now())
+
+ f2 := data.NewField("test", nil, make([]*float64, 2))
+ f2.SetConcrete(0, 5.0)
+ f2.SetConcrete(1, 20.0)
+
+ f2.Config = &data.FieldConfig{
+ Thresholds: &data.ThresholdsConfig{
+ Mode: data.ThresholdsModeAbsolute,
+ Steps: []data.Threshold{
+ {
+ Value: 10,
+ State: "normal",
+ Color: "green",
+ },
+ },
+ },
+ }
+
+ frame := data.NewFrame("test", f1, f2)
+
+ channelFrames, err := outputter.Output(context.Background(), OutputVars{}, frame)
+ require.NoError(t, err)
+ require.Len(t, channelFrames, 1)
+
+ stateFrame := channelFrames[0].Frame
+
+ require.Len(t, stateFrame.Fields, 4)
+ require.Equal(t, 5.0, stateFrame.Fields[1].At(0))
+ require.Equal(t, "", stateFrame.Fields[2].At(0))
+ require.Equal(t, "", stateFrame.Fields[3].At(0))
+
+ require.Equal(t, 20.0, stateFrame.Fields[1].At(1))
+ require.Equal(t, "normal", stateFrame.Fields[2].At(1))
+ require.Equal(t, "green", stateFrame.Fields[3].At(1))
+}
+
+func TestThresholdOutput_WithPreviousFrame_SingleRow(t *testing.T) {
+ mockCtrl := gomock.NewController(t)
+ defer mockCtrl.Finish()
+
+ mockStorage := NewMockFrameGetSetter(mockCtrl)
+
+ mockStorage.EXPECT().Get(gomock.Any(), gomock.Any()).DoAndReturn(func(orgID int64, channel string) (*data.Frame, bool, error) {
+ f1 := data.NewField("time", nil, make([]time.Time, 1))
+ f1.Set(0, time.Now())
+ f2 := data.NewField("test", nil, make([]*float64, 1))
+ f2.SetConcrete(0, 20.0)
+ frame := data.NewFrame("test", f1, f2)
+ return frame, true, nil
+ }).Times(1)
+
+ mockStorage.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Times(1)
+
+ outputter := NewThresholdOutput(mockStorage, ThresholdOutputConfig{
+ FieldName: "test",
+ Channel: "stream/test/with_previous_frame",
+ })
+
+ f1 := data.NewField("time", nil, make([]time.Time, 1))
+ f1.Set(0, time.Now())
+
+ f2 := data.NewField("test", nil, make([]*float64, 1))
+ f2.SetConcrete(0, 20.0)
+
+ f2.Config = &data.FieldConfig{
+ Thresholds: &data.ThresholdsConfig{
+ Mode: data.ThresholdsModeAbsolute,
+ Steps: []data.Threshold{
+ {
+ Value: 10,
+ State: "normal",
+ Color: "green",
+ },
+ },
+ },
+ }
+
+ frame := data.NewFrame("test", f1, f2)
+
+ channelFrames, err := outputter.Output(context.Background(), OutputVars{}, frame)
+ require.NoError(t, err)
+ require.Len(t, channelFrames, 0)
+}
+
+func TestThresholdOutput_WithPreviousFrame_MultipleRows(t *testing.T) {
+ mockCtrl := gomock.NewController(t)
+ defer mockCtrl.Finish()
+
+ mockStorage := NewMockFrameGetSetter(mockCtrl)
+
+ mockStorage.EXPECT().Get(gomock.Any(), gomock.Any()).DoAndReturn(func(orgID int64, channel string) (*data.Frame, bool, error) {
+ f1 := data.NewField("time", nil, make([]time.Time, 1))
+ f1.Set(0, time.Now())
+ f2 := data.NewField("test", nil, make([]*float64, 1))
+ f2.SetConcrete(0, 20.0)
+ frame := data.NewFrame("test", f1, f2)
+ return frame, true, nil
+ }).Times(1)
+
+ mockStorage.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any()).Times(1)
+
+ outputter := NewThresholdOutput(mockStorage, ThresholdOutputConfig{
+ FieldName: "test",
+ Channel: "stream/test/with_previous_frame",
+ })
+
+ f1 := data.NewField("time", nil, make([]time.Time, 2))
+ f1.Set(0, time.Now())
+ f1.Set(1, time.Now())
+
+ f2 := data.NewField("test", nil, make([]*float64, 2))
+ f2.SetConcrete(0, 5.0)
+ f2.SetConcrete(1, 20.0)
+
+ f2.Config = &data.FieldConfig{
+ Thresholds: &data.ThresholdsConfig{
+ Mode: data.ThresholdsModeAbsolute,
+ Steps: []data.Threshold{
+ {
+ Value: 10,
+ State: "normal",
+ Color: "green",
+ },
+ },
+ },
+ }
+
+ frame := data.NewFrame("test", f1, f2)
+
+ channelFrames, err := outputter.Output(context.Background(), OutputVars{}, frame)
+ require.NoError(t, err)
+ require.Len(t, channelFrames, 1)
+}
diff --git a/pkg/services/live/pipeline/pipeline.go b/pkg/services/live/pipeline/pipeline.go
new file mode 100644
index 00000000000..ad5efca7d0c
--- /dev/null
+++ b/pkg/services/live/pipeline/pipeline.go
@@ -0,0 +1,234 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/grafana/grafana-plugin-sdk-go/live"
+)
+
+// ChannelFrame is a wrapper over data.Frame with additional channel information.
+// Channel is used for rule routing, if the channel is empty then frame processing
+// will try to take current rule Processor and Outputter. If channel is not empty
+// then frame processing will be redirected to a corresponding channel rule.
+// TODO: avoid recursion, increment a counter while frame travels over pipeline steps, make it configurable.
+type ChannelFrame struct {
+ Channel string
+ Frame *data.Frame
+}
+
+// Vars has some helpful things pipeline entities could use.
+type Vars struct {
+ OrgID int64
+ Channel string
+ Scope string
+ Namespace string
+ Path string
+}
+
+// ProcessorVars has some helpful things Processor entities could use.
+type ProcessorVars struct {
+ Vars
+}
+
+// OutputVars has some helpful things Outputter entities could use.
+type OutputVars struct {
+ ProcessorVars
+}
+
+// Converter converts raw bytes to slice of ChannelFrame. Each element
+// of resulting slice will be then individually processed and outputted
+// according configured channel rules.
+type Converter interface {
+ Convert(ctx context.Context, vars Vars, body []byte) ([]*ChannelFrame, error)
+}
+
+// Processor can modify data.Frame in a custom way before it will be outputted.
+type Processor interface {
+ Process(ctx context.Context, vars ProcessorVars, frame *data.Frame) (*data.Frame, error)
+}
+
+// Outputter outputs data.Frame to a custom destination. Or simply
+// do nothing if some conditions not met.
+type Outputter interface {
+ Output(ctx context.Context, vars OutputVars, frame *data.Frame) ([]*ChannelFrame, error)
+}
+
+// LiveChannelRule is an in-memory representation of each specific rule, with Converter, Processor
+// and Outputter to be executed by Pipeline.
+type LiveChannelRule struct {
+ OrgId int64
+ Pattern string
+ Converter Converter
+ Processor Processor
+ Outputter Outputter
+}
+
+// Label ...
+type Label struct {
+ Name string `json:"name"`
+ Value string `json:"value"` // Can be JSONPath or Goja script.
+}
+
+// Field description.
+type Field struct {
+ Name string `json:"name"`
+ Type data.FieldType `json:"type"`
+ Value string `json:"value"` // Can be JSONPath or Goja script.
+ Labels []Label `json:"labels,omitempty"`
+ Config *data.FieldConfig `json:"config,omitempty"`
+}
+
+type ChannelRuleGetter interface {
+ Get(orgID int64, channel string) (*LiveChannelRule, bool, error)
+}
+
+// Pipeline allows processing custom input data according to user-defined rules.
+// This includes:
+// * transforming custom input to data.Frame objects
+// * do some processing on these frames
+// * output resulting frames to various destinations.
+type Pipeline struct {
+ ruleGetter ChannelRuleGetter
+}
+
+// New creates new Pipeline.
+func New(ruleGetter ChannelRuleGetter) (*Pipeline, error) {
+ logger.Info("Live pipeline initialization")
+ p := &Pipeline{
+ ruleGetter: ruleGetter,
+ }
+ if os.Getenv("GF_LIVE_PIPELINE_DEV") != "" {
+ go postTestData() // TODO: temporary for development, remove before merge.
+ }
+ return p, nil
+}
+
+func (p *Pipeline) Get(orgID int64, channel string) (*LiveChannelRule, bool, error) {
+ return p.ruleGetter.Get(orgID, channel)
+}
+
+func (p *Pipeline) ProcessInput(ctx context.Context, orgID int64, channelID string, body []byte) (bool, error) {
+ rule, ok, err := p.ruleGetter.Get(orgID, channelID)
+ if err != nil {
+ return false, err
+ }
+ if !ok {
+ return false, nil
+ }
+ channelFrames, ok, err := p.dataToChannelFrames(ctx, *rule, orgID, channelID, body)
+ if err != nil {
+ return false, err
+ }
+ if !ok {
+ return false, nil
+ }
+ err = p.processChannelFrames(ctx, orgID, channelID, channelFrames)
+ if err != nil {
+ return false, fmt.Errorf("error processing frame: %w", err)
+ }
+ return true, nil
+}
+
+func (p *Pipeline) dataToChannelFrames(ctx context.Context, rule LiveChannelRule, orgID int64, channelID string, body []byte) ([]*ChannelFrame, bool, error) {
+ if rule.Converter == nil {
+ return nil, false, nil
+ }
+
+ channel, err := live.ParseChannel(channelID)
+ if err != nil {
+ logger.Error("Error parsing channel", "error", err, "channel", channelID)
+ return nil, false, err
+ }
+
+ vars := Vars{
+ OrgID: orgID,
+ Channel: channelID,
+ Scope: channel.Scope,
+ Namespace: channel.Namespace,
+ Path: channel.Path,
+ }
+
+ frames, err := rule.Converter.Convert(ctx, vars, body)
+ if err != nil {
+ logger.Error("Error converting data", "error", err)
+ return nil, false, err
+ }
+
+ return frames, true, nil
+}
+
+func (p *Pipeline) processChannelFrames(ctx context.Context, orgID int64, channelID string, channelFrames []*ChannelFrame) error {
+ for _, channelFrame := range channelFrames {
+ var processorChannel = channelID
+ if channelFrame.Channel != "" {
+ processorChannel = channelFrame.Channel
+ }
+ err := p.processFrame(ctx, orgID, processorChannel, channelFrame.Frame)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (p *Pipeline) processFrame(ctx context.Context, orgID int64, channelID string, frame *data.Frame) error {
+ rule, ruleOk, err := p.ruleGetter.Get(orgID, channelID)
+ if err != nil {
+ logger.Error("Error getting rule", "error", err)
+ return err
+ }
+ if !ruleOk {
+ logger.Debug("Rule not found", "channel", channelID)
+ return nil
+ }
+
+ ch, err := live.ParseChannel(channelID)
+ if err != nil {
+ logger.Error("Error parsing channel", "error", err, "channel", channelID)
+ return err
+ }
+
+ vars := ProcessorVars{
+ Vars: Vars{
+ OrgID: orgID,
+ Channel: channelID,
+ Scope: ch.Scope,
+ Namespace: ch.Namespace,
+ Path: ch.Path,
+ },
+ }
+
+ if rule.Processor != nil {
+ frame, err = rule.Processor.Process(ctx, vars, frame)
+ if err != nil {
+ logger.Error("Error processing frame", "error", err)
+ return err
+ }
+ if frame == nil {
+ return nil
+ }
+ }
+
+ outputVars := OutputVars{
+ ProcessorVars: vars,
+ }
+
+ if rule.Outputter != nil {
+ frames, err := rule.Outputter.Output(ctx, outputVars, frame)
+ if err != nil {
+ logger.Error("Error outputting frame", "error", err)
+ return err
+ }
+ if len(frames) > 0 {
+ err := p.processChannelFrames(ctx, vars.OrgID, vars.Channel, frames)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
diff --git a/pkg/services/live/pipeline/pipeline_test.go b/pkg/services/live/pipeline/pipeline_test.go
new file mode 100644
index 00000000000..0313a10a430
--- /dev/null
+++ b/pkg/services/live/pipeline/pipeline_test.go
@@ -0,0 +1,107 @@
+package pipeline
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "testing"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+
+ "github.com/stretchr/testify/require"
+)
+
+type testRuleGetter struct {
+ mu sync.Mutex
+ rules map[string]*LiveChannelRule
+}
+
+func (t *testRuleGetter) Get(orgID int64, channel string) (*LiveChannelRule, bool, error) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ rule, ok := t.rules[channel]
+ return rule, ok, nil
+}
+
+func TestPipeline_New(t *testing.T) {
+ p, err := New(&testRuleGetter{})
+ require.NoError(t, err)
+ require.NotNil(t, p)
+}
+
+func TestPipelineNoConverter(t *testing.T) {
+ p, err := New(&testRuleGetter{
+ rules: map[string]*LiveChannelRule{
+ "test": {
+ Converter: nil,
+ },
+ },
+ })
+ require.NoError(t, err)
+ ok, err := p.ProcessInput(context.Background(), 1, "test", []byte(`{}`))
+ require.NoError(t, err)
+ require.False(t, ok)
+}
+
+type testConverter struct {
+ channel string
+ frame *data.Frame
+}
+
+func (t *testConverter) Convert(_ context.Context, _ Vars, _ []byte) ([]*ChannelFrame, error) {
+ return []*ChannelFrame{{Channel: t.channel, Frame: t.frame}}, nil
+}
+
+type testProcessor struct{}
+
+func (t *testProcessor) Process(_ context.Context, _ ProcessorVars, frame *data.Frame) (*data.Frame, error) {
+ return frame, nil
+}
+
+type testOutputter struct {
+ err error
+ frame *data.Frame
+}
+
+func (t *testOutputter) Output(_ context.Context, _ OutputVars, frame *data.Frame) ([]*ChannelFrame, error) {
+ if t.err != nil {
+ return nil, t.err
+ }
+ t.frame = frame
+ return nil, nil
+}
+
+func TestPipeline(t *testing.T) {
+ outputter := &testOutputter{}
+ p, err := New(&testRuleGetter{
+ rules: map[string]*LiveChannelRule{
+ "stream/test/xxx": {
+ Converter: &testConverter{"", data.NewFrame("test")},
+ Processor: &testProcessor{},
+ Outputter: outputter,
+ },
+ },
+ })
+ require.NoError(t, err)
+ ok, err := p.ProcessInput(context.Background(), 1, "stream/test/xxx", []byte(`{}`))
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.NotNil(t, outputter.frame)
+}
+
+func TestPipeline_OutputError(t *testing.T) {
+ boomErr := errors.New("boom")
+ outputter := &testOutputter{err: boomErr}
+ p, err := New(&testRuleGetter{
+ rules: map[string]*LiveChannelRule{
+ "stream/test/xxx": {
+ Converter: &testConverter{"", data.NewFrame("test")},
+ Processor: &testProcessor{},
+ Outputter: outputter,
+ },
+ },
+ })
+ require.NoError(t, err)
+ _, err = p.ProcessInput(context.Background(), 1, "stream/test/xxx", []byte(`{}`))
+ require.ErrorIs(t, err, boomErr)
+}
diff --git a/pkg/services/live/pipeline/processor_drop_field.go b/pkg/services/live/pipeline/processor_drop_field.go
new file mode 100644
index 00000000000..536579ad269
--- /dev/null
+++ b/pkg/services/live/pipeline/processor_drop_field.go
@@ -0,0 +1,37 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type DropFieldsProcessorConfig struct {
+ FieldNames []string `json:"fieldNames"`
+}
+
+// DropFieldsProcessor can drop specified fields from a data.Frame.
+type DropFieldsProcessor struct {
+ config DropFieldsProcessorConfig
+}
+
+func removeIndex(s []*data.Field, index int) []*data.Field {
+ return append(s[:index], s[index+1:]...)
+}
+
+func NewDropFieldsProcessor(config DropFieldsProcessorConfig) *DropFieldsProcessor {
+ return &DropFieldsProcessor{config: config}
+}
+
+func (d DropFieldsProcessor) Process(_ context.Context, _ ProcessorVars, frame *data.Frame) (*data.Frame, error) {
+ for _, f := range d.config.FieldNames {
+ inner:
+ for i, field := range frame.Fields {
+ if f == field.Name {
+ frame.Fields = removeIndex(frame.Fields, i)
+ continue inner
+ }
+ }
+ }
+ return frame, nil
+}
diff --git a/pkg/services/live/pipeline/processor_keep_field.go b/pkg/services/live/pipeline/processor_keep_field.go
new file mode 100644
index 00000000000..e71c411c729
--- /dev/null
+++ b/pkg/services/live/pipeline/processor_keep_field.go
@@ -0,0 +1,40 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+type KeepFieldsProcessorConfig struct {
+ FieldNames []string `json:"fieldNames"`
+}
+
+// KeepFieldsProcessor can keep specified fields in a data.Frame dropping all other fields.
+type KeepFieldsProcessor struct {
+ config KeepFieldsProcessorConfig
+}
+
+func NewKeepFieldsProcessor(config KeepFieldsProcessorConfig) *KeepFieldsProcessor {
+ return &KeepFieldsProcessor{config: config}
+}
+
+func stringInSlice(str string, slice []string) bool {
+ for _, s := range slice {
+ if s == str {
+ return true
+ }
+ }
+ return false
+}
+
+func (d KeepFieldsProcessor) Process(_ context.Context, _ ProcessorVars, frame *data.Frame) (*data.Frame, error) {
+ var fieldsToKeep []*data.Field
+ for _, field := range frame.Fields {
+ if stringInSlice(field.Name, d.config.FieldNames) {
+ fieldsToKeep = append(fieldsToKeep, field)
+ }
+ }
+ f := data.NewFrame(frame.Name, fieldsToKeep...)
+ return f, nil
+}
diff --git a/pkg/services/live/pipeline/processor_multiple.go b/pkg/services/live/pipeline/processor_multiple.go
new file mode 100644
index 00000000000..40d40798971
--- /dev/null
+++ b/pkg/services/live/pipeline/processor_multiple.go
@@ -0,0 +1,29 @@
+package pipeline
+
+import (
+ "context"
+
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// MultipleProcessor can combine several Processor and
+// execute them sequentially.
+type MultipleProcessor struct {
+ Processors []Processor
+}
+
+func (m MultipleProcessor) Process(ctx context.Context, vars ProcessorVars, frame *data.Frame) (*data.Frame, error) {
+ for _, p := range m.Processors {
+ var err error
+ frame, err = p.Process(ctx, vars, frame)
+ if err != nil {
+ logger.Error("Error processing frame", "error", err)
+ return nil, err
+ }
+ }
+ return frame, nil
+}
+
+func NewMultipleProcessor(processors ...Processor) *MultipleProcessor {
+ return &MultipleProcessor{Processors: processors}
+}
diff --git a/pkg/services/live/pipeline/rule_builder.go b/pkg/services/live/pipeline/rule_builder.go
new file mode 100644
index 00000000000..84a4291fa55
--- /dev/null
+++ b/pkg/services/live/pipeline/rule_builder.go
@@ -0,0 +1,8 @@
+package pipeline
+
+import "context"
+
+// RuleBuilder constructs in-memory representation of channel rules.
+type RuleBuilder interface {
+ BuildRules(ctx context.Context, orgID int64) ([]*LiveChannelRule, error)
+}
diff --git a/pkg/services/live/pipeline/rule_cache_segmented.go b/pkg/services/live/pipeline/rule_cache_segmented.go
new file mode 100644
index 00000000000..26bb1fcb4fa
--- /dev/null
+++ b/pkg/services/live/pipeline/rule_cache_segmented.go
@@ -0,0 +1,83 @@
+package pipeline
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/grafana/grafana/pkg/services/live/pipeline/tree"
+)
+
+// CacheSegmentedTree provides a fast access to channel rule configuration.
+type CacheSegmentedTree struct {
+ radixMu sync.RWMutex
+ radix map[int64]*tree.Node
+ ruleBuilder RuleBuilder
+}
+
+func NewCacheSegmentedTree(storage RuleBuilder) *CacheSegmentedTree {
+ s := &CacheSegmentedTree{
+ radix: map[int64]*tree.Node{},
+ ruleBuilder: storage,
+ }
+ go s.updatePeriodically()
+ return s
+}
+
+func (s *CacheSegmentedTree) updatePeriodically() {
+ for {
+ var orgIDs []int64
+ s.radixMu.Lock()
+ for orgID := range s.radix {
+ orgIDs = append(orgIDs, orgID)
+ }
+ s.radixMu.Unlock()
+ for _, orgID := range orgIDs {
+ err := s.fillOrg(orgID)
+ if err != nil {
+ logger.Error("error filling orgId", "error", err, "orgId", orgID)
+ }
+ }
+ time.Sleep(20 * time.Second)
+ }
+}
+
+func (s *CacheSegmentedTree) fillOrg(orgID int64) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ channels, err := s.ruleBuilder.BuildRules(ctx, orgID)
+ if err != nil {
+ return err
+ }
+ s.radixMu.Lock()
+ defer s.radixMu.Unlock()
+ s.radix[orgID] = tree.New()
+ for _, ch := range channels {
+ s.radix[orgID].AddRoute("/"+ch.Pattern, ch)
+ }
+ return nil
+}
+
+func (s *CacheSegmentedTree) Get(orgID int64, channel string) (*LiveChannelRule, bool, error) {
+ s.radixMu.RLock()
+ _, ok := s.radix[orgID]
+ s.radixMu.RUnlock()
+ if !ok {
+ err := s.fillOrg(orgID)
+ if err != nil {
+ return nil, false, err
+ }
+ }
+ s.radixMu.RLock()
+ defer s.radixMu.RUnlock()
+ t, ok := s.radix[orgID]
+ if !ok {
+ return nil, false, nil
+ }
+ ps := make(tree.Params, 0, 20)
+ nodeValue := t.GetValue("/"+channel, &ps, true)
+ if nodeValue.Handler == nil {
+ return nil, false, nil
+ }
+ return nodeValue.Handler.(*LiveChannelRule), true, nil
+}
diff --git a/pkg/services/live/pipeline/rule_cache_segmented_test.go b/pkg/services/live/pipeline/rule_cache_segmented_test.go
new file mode 100644
index 00000000000..9b227ab2912
--- /dev/null
+++ b/pkg/services/live/pipeline/rule_cache_segmented_test.go
@@ -0,0 +1,56 @@
+package pipeline
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+type testBuilder struct{}
+
+func (t *testBuilder) BuildRules(_ context.Context, _ int64) ([]*LiveChannelRule, error) {
+ return []*LiveChannelRule{
+ {
+ OrgId: 1,
+ Pattern: "stream/telegraf/cpu",
+ Converter: NewAutoJsonConverter(AutoJsonConverterConfig{}),
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/telegraf/:metric",
+ },
+ {
+ OrgId: 1,
+ Pattern: "stream/telegraf/:metric/:extra",
+ },
+ }, nil
+}
+
+func TestStorage_Get(t *testing.T) {
+ s := NewCacheSegmentedTree(&testBuilder{})
+ rule, ok, err := s.Get(1, "stream/telegraf/cpu")
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.NotNil(t, rule.Converter)
+
+ rule, ok, err = s.Get(1, "stream/telegraf/mem")
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.Nil(t, rule.Converter)
+
+ rule, ok, err = s.Get(1, "stream/telegraf/mem/rss")
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.Nil(t, rule.Converter)
+}
+
+func BenchmarkRuleGet(b *testing.B) {
+ s := NewCacheSegmentedTree(&testBuilder{})
+ for i := 0; i < b.N; i++ {
+ _, ok, err := s.Get(1, "stream/telegraf/cpu")
+ if err != nil || !ok {
+ b.Fatal("unexpected return values")
+ }
+ }
+}
diff --git a/pkg/services/live/pipeline/storage_file.go b/pkg/services/live/pipeline/storage_file.go
new file mode 100644
index 00000000000..2ab33a6efae
--- /dev/null
+++ b/pkg/services/live/pipeline/storage_file.go
@@ -0,0 +1,43 @@
+package pipeline
+
+import (
+ "context"
+ "encoding/json"
+ "io/ioutil"
+ "os"
+)
+
+// FileStorage can load channel rules from a file on disk.
+type FileStorage struct{}
+
+func (f *FileStorage) ListRemoteWriteBackends(_ context.Context, orgID int64) ([]RemoteWriteBackend, error) {
+ backendBytes, _ := ioutil.ReadFile(os.Getenv("GF_LIVE_REMOTE_WRITE_BACKENDS_FILE"))
+ var remoteWriteBackends RemoteWriteBackends
+ err := json.Unmarshal(backendBytes, &remoteWriteBackends)
+ if err != nil {
+ return nil, err
+ }
+ var backends []RemoteWriteBackend
+ for _, b := range remoteWriteBackends.Backends {
+ if b.OrgId == orgID || (orgID == 1 && b.OrgId == 0) {
+ backends = append(backends, b)
+ }
+ }
+ return backends, nil
+}
+
+func (f *FileStorage) ListChannelRules(_ context.Context, orgID int64) ([]ChannelRule, error) {
+ ruleBytes, _ := ioutil.ReadFile(os.Getenv("GF_LIVE_CHANNEL_RULES_FILE"))
+ var channelRules ChannelRules
+ err := json.Unmarshal(ruleBytes, &channelRules)
+ if err != nil {
+ return nil, err
+ }
+ var rules []ChannelRule
+ for _, r := range channelRules.Rules {
+ if r.OrgId == orgID || (orgID == 1 && r.OrgId == 0) {
+ rules = append(rules, r)
+ }
+ }
+ return rules, nil
+}
diff --git a/pkg/services/live/pipeline/testdata/json_auto.golden.txt b/pkg/services/live/pipeline/testdata/json_auto.golden.txt
new file mode 100644
index 00000000000..5421fcd73ea
--- /dev/null
+++ b/pkg/services/live/pipeline/testdata/json_auto.golden.txt
@@ -0,0 +1,16 @@
+🌟 This was machine generated. Do not edit. 🌟
+
+Frame[0]
+Name:
+Dimensions: 8 Fields by 1 Rows
++-------------------------------+------------------+-----------------------+-----------------------+--------------------+--------------------+----------------------------+----------------------------+
+| Name: | Name: ax | Name: string_array[0] | Name: string_array[1] | Name: int_array[0] | Name: int_array[1] | Name: map_with_floats.key1 | Name: map_with_floats.key2 |
+| Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
+| Type: []time.Time | Type: []*float64 | Type: []*string | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*float64 | Type: []*float64 |
++-------------------------------+------------------+-----------------------+-----------------------+--------------------+--------------------+----------------------------+----------------------------+
+| 2021-01-01 12:12:12 +0000 UTC | 1 | 1 | 2 | 1 | 2 | 2 | 3 |
++-------------------------------+------------------+-----------------------+-----------------------+--------------------+--------------------+----------------------------+----------------------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////KAQAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAABc/P//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAHz8//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAACAAAAEADAADUAgAAUAIAAOQBAAB0AQAABAEAAIQAAAAEAAAAWv3//xQAAABMAAAATAAAAAAAAwFMAAAAAQAAAAQAAADg/P//CAAAACAAAAAUAAAAbWFwX3dpdGhfZmxvYXRzLmtleTIAAAAABAAAAG5hbWUAAAAAAAAAAOr8//8AAAIAFAAAAG1hcF93aXRoX2Zsb2F0cy5rZXkyAAAAANb9//8UAAAATAAAAEwAAAAAAAMBTAAAAAEAAAAEAAAAXP3//wgAAAAgAAAAFAAAAG1hcF93aXRoX2Zsb2F0cy5rZXkxAAAAAAQAAABuYW1lAAAAAAAAAABm/f//AAACABQAAABtYXBfd2l0aF9mbG9hdHMua2V5MQAAAABS/v//FAAAAEQAAABEAAAAAAADAUQAAAABAAAABAAAANj9//8IAAAAGAAAAAwAAABpbnRfYXJyYXlbMV0AAAAABAAAAG5hbWUAAAAAAAAAANr9//8AAAIADAAAAGludF9hcnJheVsxXQAAAAC+/v//FAAAAEQAAABEAAAAAAADAUQAAAABAAAABAAAAET+//8IAAAAGAAAAAwAAABpbnRfYXJyYXlbMF0AAAAABAAAAG5hbWUAAAAAAAAAAEb+//8AAAIADAAAAGludF9hcnJheVswXQAAAAAq////FAAAAEQAAABEAAAAAAAFAUAAAAABAAAABAAAALD+//8IAAAAGAAAAA8AAABzdHJpbmdfYXJyYXlbMV0ABAAAAG5hbWUAAAAAAAAAAJj///8PAAAAc3RyaW5nX2FycmF5WzFdAJL///8UAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAEAAAAGP///wgAAAAYAAAADwAAAHN0cmluZ19hcnJheVswXQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAPAAAAc3RyaW5nX2FycmF5WzBdAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAAA4AAAAOAAAAAAAAwE4AAAAAQAAAAQAAACY////CAAAAAwAAAACAAAAYXgAAAQAAABuYW1lAAAAAAAAAACO////AAACAAIAAABheAAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEAAAABIAAAAAAAACkgAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAAAAAAAAAAA//////gBAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAABQAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAA4AQAAAQAAAAAAAAAAAAAAEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAABgAAAAAAAAACAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAIAAAAAAAAACgAAAAAAAAACAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAIAAAAAAAAADgAAAAAAAAAAAAAAAAAAAA4AAAAAAAAAAgAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAACAAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAIAAAAAAAAAAAAAAAIAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAGL0vkhpWFgAAAAAAAPA/AAAAAAEAAAAxAAAAAAAAAAAAAAABAAAAMgAAAAAAAAAAAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQBAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA8AAAAAAADAAEAAAA4BAAAAAAAAAACAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAABc/P//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAHz8//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAACAAAAEADAADUAgAAUAIAAOQBAAB0AQAABAEAAIQAAAAEAAAAWv3//xQAAABMAAAATAAAAAAAAwFMAAAAAQAAAAQAAADg/P//CAAAACAAAAAUAAAAbWFwX3dpdGhfZmxvYXRzLmtleTIAAAAABAAAAG5hbWUAAAAAAAAAAOr8//8AAAIAFAAAAG1hcF93aXRoX2Zsb2F0cy5rZXkyAAAAANb9//8UAAAATAAAAEwAAAAAAAMBTAAAAAEAAAAEAAAAXP3//wgAAAAgAAAAFAAAAG1hcF93aXRoX2Zsb2F0cy5rZXkxAAAAAAQAAABuYW1lAAAAAAAAAABm/f//AAACABQAAABtYXBfd2l0aF9mbG9hdHMua2V5MQAAAABS/v//FAAAAEQAAABEAAAAAAADAUQAAAABAAAABAAAANj9//8IAAAAGAAAAAwAAABpbnRfYXJyYXlbMV0AAAAABAAAAG5hbWUAAAAAAAAAANr9//8AAAIADAAAAGludF9hcnJheVsxXQAAAAC+/v//FAAAAEQAAABEAAAAAAADAUQAAAABAAAABAAAAET+//8IAAAAGAAAAAwAAABpbnRfYXJyYXlbMF0AAAAABAAAAG5hbWUAAAAAAAAAAEb+//8AAAIADAAAAGludF9hcnJheVswXQAAAAAq////FAAAAEQAAABEAAAAAAAFAUAAAAABAAAABAAAALD+//8IAAAAGAAAAA8AAABzdHJpbmdfYXJyYXlbMV0ABAAAAG5hbWUAAAAAAAAAAJj///8PAAAAc3RyaW5nX2FycmF5WzFdAJL///8UAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAEAAAAGP///wgAAAAYAAAADwAAAHN0cmluZ19hcnJheVswXQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAPAAAAc3RyaW5nX2FycmF5WzBdAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAAA4AAAAOAAAAAAAAwE4AAAAAQAAAAQAAACY////CAAAAAwAAAACAAAAYXgAAAQAAABuYW1lAAAAAAAAAACO////AAACAAIAAABheAAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEAAAABIAAAAAAAACkgAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAAAAAAAAAAAWAQAAEFSUk9XMQ==
diff --git a/pkg/services/live/pipeline/testdata/json_auto.json b/pkg/services/live/pipeline/testdata/json_auto.json
new file mode 100644
index 00000000000..740f8d8f198
--- /dev/null
+++ b/pkg/services/live/pipeline/testdata/json_auto.json
@@ -0,0 +1,10 @@
+{
+ "ax": 1,
+ "string_array": ["1", "2"],
+ "int_array": [1, 2],
+ "map_with_floats": {
+ "key1": 2.0,
+ "key2": 3.0
+ },
+ "bx": null
+}
diff --git a/pkg/services/live/pipeline/testdata/json_exact.golden.txt b/pkg/services/live/pipeline/testdata/json_exact.golden.txt
new file mode 100644
index 00000000000..8a27aa940d1
--- /dev/null
+++ b/pkg/services/live/pipeline/testdata/json_exact.golden.txt
@@ -0,0 +1,16 @@
+🌟 This was machine generated. Do not edit. 🌟
+
+Frame[0]
+Name:
+Dimensions: 3 Fields by 1 Rows
++-------------------------------+------------------+----------------------------+
+| Name: time | Name: ax | Name: key1 |
+| Labels: | Labels: | Labels: label1=3, label2=3 |
+| Type: []time.Time | Type: []*float64 | Type: []*float64 |
++-------------------------------+------------------+----------------------------+
+| 2021-01-01 12:12:12 +0000 UTC | 1 | 2 |
++-------------------------------+------------------+----------------------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAAA0/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFT+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAwAAAEABAACwAAAABAAAAGr///8UAAAAeAAAAHgAAAAAAAMBeAAAAAIAAAAsAAAABAAAAKj+//8IAAAAEAAAAAQAAABrZXkxAAAAAAQAAABuYW1lAAAAAMz+//8IAAAAJAAAABsAAAB7ImxhYmVsMSI6IjMiLCJsYWJlbDIiOiIzIn0ABgAAAGxhYmVscwAAAAAAANr+//8AAAIABAAAAGtleTEAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAXAAAAFwAAAAAAAMBXAAAAAIAAAAoAAAABAAAAFD///8IAAAADAAAAAIAAABheAAABAAAAG5hbWUAAAAAcP///wgAAAAMAAAAAgAAAHt9AAAGAAAAbGFiZWxzAAAAAAAAZv///wAAAgACAAAAYXgAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABoAAAAcAAAAAAAAApwAAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAACAAMAAgABAAIAAAACAAAAAwAAAACAAAAe30AAAYAAABsYWJlbHMAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAHRpbWUAAAAAAAAAAP/////oAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAGAAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAeAAAAAEAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAACAAAAAAAAAAAAAAAAwAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAYvS+SGlYWAAAAAAAA8D8AAAAAAAAAQBAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAABoAgAAAAAAAPAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAADT+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAVP7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAADAAAAQAEAALAAAAAEAAAAav///xQAAAB4AAAAeAAAAAAAAwF4AAAAAgAAACwAAAAEAAAAqP7//wgAAAAQAAAABAAAAGtleTEAAAAABAAAAG5hbWUAAAAAzP7//wgAAAAkAAAAGwAAAHsibGFiZWwxIjoiMyIsImxhYmVsMiI6IjMifQAGAAAAbGFiZWxzAAAAAAAA2v7//wAAAgAEAAAAa2V5MQAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABcAAAAXAAAAAAAAwFcAAAAAgAAACgAAAAEAAAAUP///wgAAAAMAAAAAgAAAGF4AAAEAAAAbmFtZQAAAABw////CAAAAAwAAAACAAAAe30AAAYAAABsYWJlbHMAAAAAAABm////AAACAAIAAABheAAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAGgAAABwAAAAAAAACnAAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAbmFtZQAAAAAIAAwACAAEAAgAAAAIAAAADAAAAAIAAAB7fQAABgAAAGxhYmVscwAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAACAAgAAQVJST1cx
diff --git a/pkg/services/live/pipeline/testdata/json_exact.json b/pkg/services/live/pipeline/testdata/json_exact.json
new file mode 100644
index 00000000000..740f8d8f198
--- /dev/null
+++ b/pkg/services/live/pipeline/testdata/json_exact.json
@@ -0,0 +1,10 @@
+{
+ "ax": 1,
+ "string_array": ["1", "2"],
+ "int_array": [1, 2],
+ "map_with_floats": {
+ "key1": 2.0,
+ "key2": 3.0
+ },
+ "bx": null
+}
diff --git a/pkg/services/live/pipeline/tree/LICENSE b/pkg/services/live/pipeline/tree/LICENSE
new file mode 100644
index 00000000000..875308f5235
--- /dev/null
+++ b/pkg/services/live/pipeline/tree/LICENSE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2013, Julien Schmidt
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/pkg/services/live/pipeline/tree/bytesconv.go b/pkg/services/live/pipeline/tree/bytesconv.go
new file mode 100644
index 00000000000..4173058ee4f
--- /dev/null
+++ b/pkg/services/live/pipeline/tree/bytesconv.go
@@ -0,0 +1,11 @@
+package tree
+
+// StringToBytes converts string to byte slice without a memory allocation.
+func StringToBytes(s string) []byte {
+ return []byte(s)
+}
+
+// BytesToString converts byte slice to string without a memory allocation.
+func BytesToString(b []byte) string {
+ return string(b)
+}
diff --git a/pkg/services/live/pipeline/tree/params.go b/pkg/services/live/pipeline/tree/params.go
new file mode 100644
index 00000000000..d89de754f0a
--- /dev/null
+++ b/pkg/services/live/pipeline/tree/params.go
@@ -0,0 +1,40 @@
+// Copyright 2013 Julien Schmidt. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be found
+// in the LICENSE file.
+package tree
+
+import "context"
+
+// Param is a single URL parameter, consisting of a key and a value.
+type Param struct {
+ Key string
+ Value string
+}
+
+// Params is a Param-slice, as returned by the router.
+// The slice is ordered, the first URL parameter is also the first slice value.
+// It is therefore safe to read values by the index.
+type Params []Param
+
+// Get returns the value of the first Param which key matches the given name.
+// If no matching Param is found, an empty string is returned.
+func (ps Params) Get(name string) (string, bool) {
+ for _, p := range ps {
+ if p.Key == name {
+ return p.Value, true
+ }
+ }
+ return "", false
+}
+
+type paramsKey struct{}
+
+// ParamsKey is the request context key under which URL Params are stored.
+var ParamsKey = paramsKey{}
+
+// ParamsFromContext pulls the URL parameters from a request context,
+// or returns nil if none are present.
+func ParamsFromContext(ctx context.Context) Params {
+ p, _ := ctx.Value(ParamsKey).(Params)
+ return p
+}
diff --git a/pkg/services/live/pipeline/tree/readme.md b/pkg/services/live/pipeline/tree/readme.md
new file mode 100644
index 00000000000..2066594e9c6
--- /dev/null
+++ b/pkg/services/live/pipeline/tree/readme.md
@@ -0,0 +1,9 @@
+This is a tree code from https://github.com/julienschmidt/httprouter with an important fixes made inside Gin web framework.
+
+See:
+* https://github.com/julienschmidt/httprouter/pull/329
+* https://github.com/gin-gonic/gin/issues/2786
+
+See also https://github.com/julienschmidt/httprouter/issues/235 – that's the reason why we can't use a custom branch patched with fixes.
+
+Original LICENSE and copyright left unchanged here.
diff --git a/pkg/services/live/pipeline/tree/tree.go b/pkg/services/live/pipeline/tree/tree.go
new file mode 100644
index 00000000000..2f4d1bf960a
--- /dev/null
+++ b/pkg/services/live/pipeline/tree/tree.go
@@ -0,0 +1,799 @@
+// Copyright 2013 Julien Schmidt. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be found
+// at https://github.com/julienschmidt/httprouter/blob/master/LICENSE
+
+package tree
+
+import (
+ "bytes"
+ "net/url"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+var (
+ strColon = []byte(":")
+ strStar = []byte("*")
+)
+
+func min(a, b int) int {
+ if a <= b {
+ return a
+ }
+ return b
+}
+
+func longestCommonPrefix(a, b string) int {
+ i := 0
+ max := min(len(a), len(b))
+ for i < max && a[i] == b[i] {
+ i++
+ }
+ return i
+}
+
+// addChild will add a child Node, keeping wildcards at the end
+func (n *Node) addChild(child *Node) {
+ if n.wildChild && len(n.children) > 0 {
+ wildcardChild := n.children[len(n.children)-1]
+ n.children = append(n.children[:len(n.children)-1], child, wildcardChild)
+ } else {
+ n.children = append(n.children, child)
+ }
+}
+
+func countParams(path string) uint16 {
+ var n uint16
+ s := StringToBytes(path)
+ n += uint16(bytes.Count(s, strColon))
+ n += uint16(bytes.Count(s, strStar))
+ return n
+}
+
+type nodeType uint8
+
+const (
+ root nodeType = iota + 1
+ param
+ catchAll
+)
+
+type Handler interface{}
+
+func New() *Node {
+ return new(Node)
+}
+
+type Node struct {
+ path string
+ indices string
+ wildChild bool
+ nType nodeType
+ priority uint32
+ children []*Node // child nodes, at most 1 :param style Node at the end of the array
+ handler Handler
+ fullPath string
+}
+
+// Increments priority of the given child and reorders if necessary
+func (n *Node) incrementChildPrio(pos int) int {
+ cs := n.children
+ cs[pos].priority++
+ prio := cs[pos].priority
+
+ // Adjust position (move to front)
+ newPos := pos
+ for ; newPos > 0 && cs[newPos-1].priority < prio; newPos-- {
+ // Swap Node positions
+ cs[newPos-1], cs[newPos] = cs[newPos], cs[newPos-1]
+ }
+
+ // Build new index char string
+ if newPos != pos {
+ n.indices = n.indices[:newPos] + // Unchanged prefix, might be empty
+ n.indices[pos:pos+1] + // The index char we move
+ n.indices[newPos:pos] + n.indices[pos+1:] // Rest without char at 'pos'
+ }
+
+ return newPos
+}
+
+func (n *Node) AddRoute(path string, handlers Handler) {
+ n.addRoute(path, handlers)
+}
+
+// addRoute adds a Node with the given handle to the path.
+// Not concurrency-safe!
+func (n *Node) addRoute(path string, handlers Handler) {
+ fullPath := path
+ n.priority++
+
+ // Empty tree
+ if len(n.path) == 0 && len(n.children) == 0 {
+ n.insertChild(path, fullPath, handlers)
+ n.nType = root
+ return
+ }
+
+ parentFullPathIndex := 0
+
+walk:
+ for {
+ // Find the longest common prefix.
+ // This also implies that the common prefix contains no ':' or '*'
+ // since the existing key can't contain those chars.
+ i := longestCommonPrefix(path, n.path)
+
+ // Split edge
+ if i < len(n.path) {
+ child := Node{
+ path: n.path[i:],
+ wildChild: n.wildChild,
+ indices: n.indices,
+ children: n.children,
+ handler: n.handler,
+ priority: n.priority - 1,
+ fullPath: n.fullPath,
+ }
+
+ n.children = []*Node{&child}
+ // []byte for proper unicode char conversion, see #65
+ n.indices = BytesToString([]byte{n.path[i]})
+ n.path = path[:i]
+ n.handler = nil
+ n.wildChild = false
+ n.fullPath = fullPath[:parentFullPathIndex+i]
+ }
+
+ // Make new Node a child of this Node
+ if i < len(path) {
+ path = path[i:]
+ c := path[0]
+
+ // '/' after param
+ if n.nType == param && c == '/' && len(n.children) == 1 {
+ parentFullPathIndex += len(n.path)
+ n = n.children[0]
+ n.priority++
+ continue walk
+ }
+
+ // Check if a child with the next path byte exists
+ for i, max := 0, len(n.indices); i < max; i++ {
+ if c == n.indices[i] {
+ parentFullPathIndex += len(n.path)
+ i = n.incrementChildPrio(i)
+ n = n.children[i]
+ continue walk
+ }
+ }
+
+ // Otherwise insert it
+ if c != ':' && c != '*' && n.nType != catchAll {
+ // []byte for proper unicode char conversion, see #65
+ n.indices += BytesToString([]byte{c})
+ child := &Node{
+ fullPath: fullPath,
+ }
+ n.addChild(child)
+ n.incrementChildPrio(len(n.indices) - 1)
+ n = child
+ } else if n.wildChild {
+ // inserting a wildcard Node, need to check if it conflicts with the existing wildcard
+ n = n.children[len(n.children)-1]
+ n.priority++
+
+ // Check if the wildcard matches
+ if len(path) >= len(n.path) && n.path == path[:len(n.path)] &&
+ // Adding a child to a catchAll is not possible
+ n.nType != catchAll &&
+ // Check for longer wildcard, e.g. :name and :names
+ (len(n.path) >= len(path) || path[len(n.path)] == '/') {
+ continue walk
+ }
+
+ // Wildcard conflict
+ pathSeg := path
+ if n.nType != catchAll {
+ pathSeg = strings.SplitN(pathSeg, "/", 2)[0]
+ }
+ prefix := fullPath[:strings.Index(fullPath, pathSeg)] + n.path
+ panic("'" + pathSeg +
+ "' in new path '" + fullPath +
+ "' conflicts with existing wildcard '" + n.path +
+ "' in existing prefix '" + prefix +
+ "'")
+ }
+
+ n.insertChild(path, fullPath, handlers)
+ return
+ }
+
+ // Otherwise add handle to current Node
+ if n.handler != nil {
+ panic("handler are already registered for path '" + fullPath + "'")
+ }
+ n.handler = handlers
+ n.fullPath = fullPath
+ return
+ }
+}
+
+// Search for a wildcard segment and check the name for invalid characters.
+// Returns -1 as index, if no wildcard was found.
+func findWildcard(path string) (wildcard string, i int, valid bool) {
+ // Find start
+ for start, c := range []byte(path) {
+ // A wildcard starts with ':' (param) or '*' (catch-all)
+ if c != ':' && c != '*' {
+ continue
+ }
+
+ // Find end and check for invalid characters
+ valid = true
+ for end, c := range []byte(path[start+1:]) {
+ switch c {
+ case '/':
+ return path[start : start+1+end], start, valid
+ case ':', '*':
+ valid = false
+ }
+ }
+ return path[start:], start, valid
+ }
+ return "", -1, false
+}
+
+func (n *Node) insertChild(path string, fullPath string, handlers Handler) {
+ for {
+ // Find prefix until first wildcard
+ wildcard, i, valid := findWildcard(path)
+ if i < 0 { // No wildcard found
+ break
+ }
+
+ // The wildcard name must not contain ':' and '*'
+ if !valid {
+ panic("only one wildcard per path segment is allowed, has: '" +
+ wildcard + "' in path '" + fullPath + "'")
+ }
+
+ // check if the wildcard has a name
+ if len(wildcard) < 2 {
+ panic("wildcards must be named with a non-empty name in path '" + fullPath + "'")
+ }
+
+ if wildcard[0] == ':' { // param
+ if i > 0 {
+ // Insert prefix before the current wildcard
+ n.path = path[:i]
+ path = path[i:]
+ }
+
+ child := &Node{
+ nType: param,
+ path: wildcard,
+ fullPath: fullPath,
+ }
+ n.addChild(child)
+ n.wildChild = true
+ n = child
+ n.priority++
+
+ // if the path doesn't end with the wildcard, then there
+ // will be another non-wildcard subpath starting with '/'
+ if len(wildcard) < len(path) {
+ path = path[len(wildcard):]
+
+ child := &Node{
+ priority: 1,
+ fullPath: fullPath,
+ }
+ n.addChild(child)
+ n = child
+ continue
+ }
+
+ // Otherwise we're done. Insert the handle in the new leaf
+ n.handler = handlers
+ return
+ }
+
+ // catchAll
+ if i+len(wildcard) != len(path) {
+ panic("catch-all routes are only allowed at the end of the path in path '" + fullPath + "'")
+ }
+
+ if len(n.path) > 0 && n.path[len(n.path)-1] == '/' {
+ panic("catch-all conflicts with existing handle for the path segment root in path '" + fullPath + "'")
+ }
+
+ // currently fixed width 1 for '/'
+ i--
+ if path[i] != '/' {
+ panic("no / before catch-all in path '" + fullPath + "'")
+ }
+
+ n.path = path[:i]
+
+ // First Node: catchAll Node with empty path
+ child := &Node{
+ wildChild: true,
+ nType: catchAll,
+ fullPath: fullPath,
+ }
+
+ n.addChild(child)
+ n.indices = string('/')
+ n = child
+ n.priority++
+
+ // second Node: Node holding the variable
+ child = &Node{
+ path: path[i:],
+ nType: catchAll,
+ handler: handlers,
+ priority: 1,
+ fullPath: fullPath,
+ }
+ n.children = []*Node{child}
+
+ return
+ }
+
+ // If no wildcard was found, simply insert the path and handle
+ n.path = path
+ n.handler = handlers
+ n.fullPath = fullPath
+}
+
+// NodeValue holds return values of (*Node).getValue method
+type NodeValue struct {
+ Handler Handler
+ Params *Params
+ Tsr bool
+ FullPath string
+}
+
+func (n *Node) GetValue(path string, params *Params, unescape bool) (value NodeValue) {
+ return n.getValue(path, params, unescape)
+}
+
+// Returns the handle registered with the given path (key). The values of
+// wildcards are saved to a map.
+// If no handle can be found, a TSR (trailing slash redirect) recommendation is
+// made if a handle exists with an extra (without the) trailing slash for the
+// given path.
+// nolint:gocyclo
+func (n *Node) getValue(path string, params *Params, unescape bool) (value NodeValue) {
+ var (
+ skippedPath string
+ latestNode = n // Caching the latest Node
+ )
+
+walk: // Outer loop for walking the tree
+ for {
+ prefix := n.path
+ if len(path) > len(prefix) {
+ if path[:len(prefix)] == prefix {
+ path = path[len(prefix):]
+
+ // Try all the non-wildcard children first by matching the indices
+ idxc := path[0]
+ for i, c := range []byte(n.indices) {
+ if c == idxc {
+ // strings.HasPrefix(n.children[len(n.children)-1].path, ":") == n.wildChild
+ if n.wildChild {
+ skippedPath = prefix + path
+ latestNode = &Node{
+ path: n.path,
+ wildChild: n.wildChild,
+ nType: n.nType,
+ priority: n.priority,
+ children: n.children,
+ handler: n.handler,
+ fullPath: n.fullPath,
+ }
+ }
+
+ n = n.children[i]
+ continue walk
+ }
+ }
+ // If the path at the end of the loop is not equal to '/' and the current Node has no child nodes
+ // the current Node needs to be equal to the latest matching Node
+ matched := path != "/" && !n.wildChild
+ if matched {
+ n = latestNode
+ }
+
+ // If there is no wildcard pattern, recommend a redirection
+ if !n.wildChild {
+ // Nothing found.
+ // We can recommend to redirect to the same URL without a
+ // trailing slash if a leaf exists for that path.
+ value.Tsr = path == "/" && n.handler != nil
+ return
+ }
+
+ // Handle wildcard child, which is always at the end of the array
+ n = n.children[len(n.children)-1]
+
+ switch n.nType {
+ case param:
+ // fix truncate the parameter
+ // tree_test.go line: 204
+ if matched {
+ path = prefix + path
+ // The saved path is used after the prefix route is intercepted by matching
+ if n.indices == "/" {
+ path = skippedPath[1:]
+ }
+ }
+
+ // Find param end (either '/' or path end)
+ end := 0
+ for end < len(path) && path[end] != '/' {
+ end++
+ }
+
+ // Save param value
+ if params != nil && cap(*params) > 0 {
+ if value.Params == nil {
+ value.Params = params
+ }
+ // Expand slice within preallocated capacity
+ i := len(*value.Params)
+ *value.Params = (*value.Params)[:i+1]
+ val := path[:end]
+ if unescape {
+ if v, err := url.QueryUnescape(val); err == nil {
+ val = v
+ }
+ }
+ (*value.Params)[i] = Param{
+ Key: n.path[1:],
+ Value: val,
+ }
+ }
+
+ // we need to go deeper!
+ if end < len(path) {
+ if len(n.children) > 0 {
+ path = path[end:]
+ n = n.children[0]
+ continue walk
+ }
+
+ // ... but we can't
+ value.Tsr = len(path) == end+1
+ return
+ }
+
+ if value.Handler = n.handler; value.Handler != nil {
+ value.FullPath = n.fullPath
+ return
+ }
+ if len(n.children) == 1 {
+ // No handle found. Check if a handle for this path + a
+ // trailing slash exists for TSR recommendation
+ n = n.children[0]
+ value.Tsr = n.path == "/" && n.handler != nil
+ }
+ return
+
+ case catchAll:
+ // Save param value
+ if params != nil {
+ if value.Params == nil {
+ value.Params = params
+ }
+ // Expand slice within preallocated capacity
+ i := len(*value.Params)
+ *value.Params = (*value.Params)[:i+1]
+ val := path
+ if unescape {
+ if v, err := url.QueryUnescape(path); err == nil {
+ val = v
+ }
+ }
+ (*value.Params)[i] = Param{
+ Key: n.path[2:],
+ Value: val,
+ }
+ }
+
+ value.Handler = n.handler
+ value.FullPath = n.fullPath
+ return
+
+ default:
+ panic("invalid Node type")
+ }
+ }
+ }
+
+ if path == prefix {
+ // If the current path does not equal '/' and the Node does not have a registered handle and the most recently matched Node has a child Node
+ // the current Node needs to be equal to the latest matching Node
+ if latestNode.wildChild && n.handler == nil && path != "/" {
+ n = latestNode.children[len(latestNode.children)-1]
+ }
+ // We should have reached the Node containing the handle.
+ // Check if this Node has a handle registered.
+ if value.Handler = n.handler; value.Handler != nil {
+ value.FullPath = n.fullPath
+ return
+ }
+
+ // If there is no handle for this route, but this route has a
+ // wildcard child, there must be a handle for this path with an
+ // additional trailing slash
+ if path == "/" && n.wildChild && n.nType != root {
+ value.Tsr = true
+ return
+ }
+
+ // No handle found. Check if a handle for this path + a
+ // trailing slash exists for trailing slash recommendation
+ for i, c := range []byte(n.indices) {
+ if c == '/' {
+ n = n.children[i]
+ value.Tsr = (len(n.path) == 1 && n.handler != nil) ||
+ (n.nType == catchAll && n.children[0].handler != nil)
+ return
+ }
+ }
+
+ return
+ }
+
+ if path != "/" && len(skippedPath) > 0 && strings.HasSuffix(skippedPath, path) {
+ path = skippedPath
+ // Reduce the number of cycles
+ n, latestNode = latestNode, n
+ // skippedPath cannot execute
+ // example:
+ // * /:cc/cc
+ // call /a/cc expectations:match/200 Actual:match/200
+ // call /a/dd expectations:unmatch/404 Actual: panic
+ // call /addr/dd/aa expectations:unmatch/404 Actual: panic
+ // skippedPath: It can only be executed if the secondary route is not found
+ skippedPath = ""
+ continue walk
+ }
+
+ // Nothing found. We can recommend to redirect to the same URL with an
+ // extra trailing slash if a leaf exists for that path
+ value.Tsr = path == "/" ||
+ (len(prefix) == len(path)+1 && n.handler != nil)
+ return
+ }
+}
+
+// Makes a case-insensitive lookup of the given path and tries to find a handler.
+// It can optionally also fix trailing slashes.
+// It returns the case-corrected path and a bool indicating whether the lookup
+// was successful.
+func (n *Node) findCaseInsensitivePath(path string, fixTrailingSlash bool) ([]byte, bool) {
+ const stackBufSize = 128
+
+ // Use a static sized buffer on the stack in the common case.
+ // If the path is too long, allocate a buffer on the heap instead.
+ buf := make([]byte, 0, stackBufSize)
+ if length := len(path) + 1; length > stackBufSize {
+ buf = make([]byte, 0, length)
+ }
+
+ ciPath := n.findCaseInsensitivePathRec(
+ path,
+ buf, // Preallocate enough memory for new path
+ [4]byte{}, // Empty rune buffer
+ fixTrailingSlash,
+ )
+
+ return ciPath, ciPath != nil
+}
+
+// Shift bytes in array by n bytes left
+func shiftNRuneBytes(rb [4]byte, n int) [4]byte {
+ switch n {
+ case 0:
+ return rb
+ case 1:
+ return [4]byte{rb[1], rb[2], rb[3], 0}
+ case 2:
+ return [4]byte{rb[2], rb[3]}
+ case 3:
+ return [4]byte{rb[3]}
+ default:
+ return [4]byte{}
+ }
+}
+
+// Recursive case-insensitive lookup function used by n.findCaseInsensitivePath
+// nolint:gocyclo
+func (n *Node) findCaseInsensitivePathRec(path string, ciPath []byte, rb [4]byte, fixTrailingSlash bool) []byte {
+ npLen := len(n.path)
+
+walk: // Outer loop for walking the tree
+ for len(path) >= npLen && (npLen == 0 || strings.EqualFold(path[1:npLen], n.path[1:])) {
+ // Add common prefix to result
+ oldPath := path
+ path = path[npLen:]
+ ciPath = append(ciPath, n.path...)
+
+ if len(path) == 0 {
+ // We should have reached the Node containing the handle.
+ // Check if this Node has a handle registered.
+ if n.handler != nil {
+ return ciPath
+ }
+
+ // No handle found.
+ // Try to fix the path by adding a trailing slash
+ if fixTrailingSlash {
+ for i, c := range []byte(n.indices) {
+ if c == '/' {
+ n = n.children[i]
+ if (len(n.path) == 1 && n.handler != nil) ||
+ (n.nType == catchAll && n.children[0].handler != nil) {
+ return append(ciPath, '/')
+ }
+ return nil
+ }
+ }
+ }
+ return nil
+ }
+
+ // If this Node does not have a wildcard (param or catchAll) child,
+ // we can just look up the next child Node and continue to walk down
+ // the tree
+ if !n.wildChild {
+ // Skip rune bytes already processed
+ rb = shiftNRuneBytes(rb, npLen)
+
+ if rb[0] != 0 {
+ // Old rune not finished
+ idxc := rb[0]
+ for i, c := range []byte(n.indices) {
+ if c == idxc {
+ // continue with child Node
+ n = n.children[i]
+ npLen = len(n.path)
+ continue walk
+ }
+ }
+ } else {
+ // Process a new rune
+ var rv rune
+
+ // Find rune start.
+ // Runes are up to 4 byte long,
+ // -4 would definitely be another rune.
+ var off int
+ for max := min(npLen, 3); off < max; off++ {
+ if i := npLen - off; utf8.RuneStart(oldPath[i]) {
+ // read rune from cached path
+ rv, _ = utf8.DecodeRuneInString(oldPath[i:])
+ break
+ }
+ }
+
+ // Calculate lowercase bytes of current rune
+ lo := unicode.ToLower(rv)
+ utf8.EncodeRune(rb[:], lo)
+
+ // Skip already processed bytes
+ rb = shiftNRuneBytes(rb, off)
+
+ idxc := rb[0]
+ for i, c := range []byte(n.indices) {
+ // Lowercase matches
+ if c == idxc {
+ // must use a recursive approach since both the
+ // uppercase byte and the lowercase byte might exist
+ // as an index
+ if out := n.children[i].findCaseInsensitivePathRec(
+ path, ciPath, rb, fixTrailingSlash,
+ ); out != nil {
+ return out
+ }
+ break
+ }
+ }
+
+ // If we found no match, the same for the uppercase rune,
+ // if it differs
+ if up := unicode.ToUpper(rv); up != lo {
+ utf8.EncodeRune(rb[:], up)
+ rb = shiftNRuneBytes(rb, off)
+
+ idxc := rb[0]
+ for i, c := range []byte(n.indices) {
+ // Uppercase matches
+ if c == idxc {
+ // Continue with child Node
+ n = n.children[i]
+ npLen = len(n.path)
+ continue walk
+ }
+ }
+ }
+ }
+
+ // Nothing found. We can recommend to redirect to the same URL
+ // without a trailing slash if a leaf exists for that path
+ if fixTrailingSlash && path == "/" && n.handler != nil {
+ return ciPath
+ }
+ return nil
+ }
+
+ n = n.children[0]
+ switch n.nType {
+ case param:
+ // Find param end (either '/' or path end)
+ end := 0
+ for end < len(path) && path[end] != '/' {
+ end++
+ }
+
+ // Add param value to case insensitive path
+ ciPath = append(ciPath, path[:end]...)
+
+ // We need to go deeper!
+ if end < len(path) {
+ if len(n.children) > 0 {
+ // Continue with child Node
+ n = n.children[0]
+ npLen = len(n.path)
+ path = path[end:]
+ continue
+ }
+
+ // ... but we can't
+ if fixTrailingSlash && len(path) == end+1 {
+ return ciPath
+ }
+ return nil
+ }
+
+ if n.handler != nil {
+ return ciPath
+ }
+
+ if fixTrailingSlash && len(n.children) == 1 {
+ // No handle found. Check if a handle for this path + a
+ // trailing slash exists
+ n = n.children[0]
+ if n.path == "/" && n.handler != nil {
+ return append(ciPath, '/')
+ }
+ }
+
+ return nil
+
+ case catchAll:
+ return append(ciPath, path...)
+
+ default:
+ panic("invalid Node type")
+ }
+ }
+
+ // Nothing found.
+ // Try to fix the path by adding / removing a trailing slash
+ if fixTrailingSlash {
+ if path == "/" {
+ return ciPath
+ }
+ if len(path)+1 == npLen && n.path[len(path)] == '/' &&
+ strings.EqualFold(path[1:], n.path[1:len(path)]) && n.handler != nil {
+ return append(ciPath, n.path...)
+ }
+ }
+ return nil
+}
diff --git a/pkg/services/live/pipeline/tree/tree_test.go b/pkg/services/live/pipeline/tree/tree_test.go
new file mode 100644
index 00000000000..6f248b466a5
--- /dev/null
+++ b/pkg/services/live/pipeline/tree/tree_test.go
@@ -0,0 +1,891 @@
+// Copyright 2013 Julien Schmidt. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be found
+// at https://github.com/julienschmidt/httprouter/blob/master/LICENSE
+
+package tree
+
+import (
+ "fmt"
+ "net/http"
+ "reflect"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+// Used as a workaround since we can't compare functions or their addresses
+var fakeHandlerValue string
+
+func fakeHandler(val string) Handler {
+ return func(http.ResponseWriter, *http.Request, Params) {
+ fakeHandlerValue = val
+ }
+}
+
+type testRequests []struct {
+ path string
+ nilHandler bool
+ route string
+ ps Params
+}
+
+func getParams() *Params {
+ ps := make(Params, 0, 20)
+ return &ps
+}
+
+func checkRequests(t *testing.T, tree *Node, requests testRequests, unescapes ...bool) {
+ unescape := false
+ if len(unescapes) >= 1 {
+ unescape = unescapes[0]
+ }
+
+ for _, request := range requests {
+ value := tree.getValue(request.path, getParams(), unescape)
+
+ if value.Handler == nil {
+ if !request.nilHandler {
+ t.Errorf("handle mismatch for route '%s': Expected non-nil handle", request.path)
+ }
+ } else if request.nilHandler {
+ t.Errorf("handle mismatch for route '%s': Expected nil handle", request.path)
+ } else {
+ handler, ok := value.Handler.(func(http.ResponseWriter, *http.Request, Params))
+ if !ok {
+ t.Errorf("invalid handler type for route '%s': %T", request.path, value.Handler)
+ }
+ handler(nil, nil, nil)
+ if fakeHandlerValue != request.route {
+ t.Errorf("handle mismatch for route '%s': Wrong handle (%s != %s)", request.path, fakeHandlerValue, request.route)
+ }
+ }
+
+ if value.Params != nil {
+ if !reflect.DeepEqual(*value.Params, request.ps) {
+ t.Errorf("Params mismatch for route '%s'", request.path)
+ }
+ }
+ }
+}
+
+func checkPriorities(t *testing.T, n *Node) uint32 {
+ var prio uint32
+ for i := range n.children {
+ prio += checkPriorities(t, n.children[i])
+ }
+
+ if n.handler != nil {
+ prio++
+ }
+
+ if n.priority != prio {
+ t.Errorf(
+ "priority mismatch for Node '%s': is %d, should be %d",
+ n.path, n.priority, prio,
+ )
+ }
+
+ return prio
+}
+
+func TestCountParams(t *testing.T) {
+ if countParams("/path/:param1/static/*catch-all") != 2 {
+ t.Fail()
+ }
+ if countParams(strings.Repeat("/:param", 256)) != 256 {
+ t.Fail()
+ }
+}
+
+func TestTreeAddAndGet(t *testing.T) {
+ tree := &Node{}
+
+ routes := [...]string{
+ "/hi",
+ "/contact",
+ "/co",
+ "/c",
+ "/a",
+ "/ab",
+ "/doc/",
+ "/doc/go_faq.html",
+ "/doc/go1.html",
+ "/α",
+ "/β",
+ }
+ for _, route := range routes {
+ tree.addRoute(route, fakeHandler(route))
+ }
+
+ checkRequests(t, tree, testRequests{
+ {"/a", false, "/a", nil},
+ {"/", true, "", nil},
+ {"/hi", false, "/hi", nil},
+ {"/contact", false, "/contact", nil},
+ {"/co", false, "/co", nil},
+ {"/con", true, "", nil}, // key mismatch
+ {"/cona", true, "", nil}, // key mismatch
+ {"/no", true, "", nil}, // no matching child
+ {"/ab", false, "/ab", nil},
+ {"/α", false, "/α", nil},
+ {"/β", false, "/β", nil},
+ })
+
+ checkPriorities(t, tree)
+}
+
+func TestTreeWildcard(t *testing.T) {
+ tree := &Node{}
+
+ routes := [...]string{
+ "/",
+ "/cmd/:tool/",
+ "/cmd/:tool/:sub",
+ "/cmd/whoami",
+ "/cmd/whoami/root",
+ "/cmd/whoami/root/",
+ "/src/*filepath",
+ "/search/",
+ "/search/:query",
+ "/search/gin-gonic",
+ "/search/google",
+ "/user_:name",
+ "/user_:name/about",
+ "/files/:dir/*filepath",
+ "/doc/",
+ "/doc/go_faq.html",
+ "/doc/go1.html",
+ "/info/:user/public",
+ "/info/:user/project/:project",
+ "/info/:user/project/golang",
+ "/aa/*xx",
+ "/ab/*xx",
+ "/:cc",
+ "/:cc/cc",
+ "/:cc/:dd/ee",
+ "/:cc/:dd/:ee/ff",
+ "/:cc/:dd/:ee/:ff/gg",
+ "/:cc/:dd/:ee/:ff/:gg/hh",
+ "/get/test/abc/",
+ "/get/:param/abc/",
+ "/something/:paramname/thirdthing",
+ "/something/secondthing/test",
+ "/get/abc",
+ "/get/:param",
+ "/get/abc/123abc",
+ "/get/abc/:param",
+ "/get/abc/123abc/xxx8",
+ "/get/abc/123abc/:param",
+ "/get/abc/123abc/xxx8/1234",
+ "/get/abc/123abc/xxx8/:param",
+ "/get/abc/123abc/xxx8/1234/ffas",
+ "/get/abc/123abc/xxx8/1234/:param",
+ "/get/abc/123abc/xxx8/1234/kkdd/12c",
+ "/get/abc/123abc/xxx8/1234/kkdd/:param",
+ "/get/abc/:param/test",
+ "/get/abc/123abd/:param",
+ "/get/abc/123abddd/:param",
+ "/get/abc/123/:param",
+ "/get/abc/123abg/:param",
+ "/get/abc/123abf/:param",
+ "/get/abc/123abfff/:param",
+ }
+ for _, route := range routes {
+ tree.addRoute(route, fakeHandler(route))
+ }
+
+ checkRequests(t, tree, testRequests{
+ {"/", false, "/", nil},
+ {"/cmd/test", true, "/cmd/:tool/", Params{Param{"tool", "test"}}},
+ {"/cmd/test/", false, "/cmd/:tool/", Params{Param{"tool", "test"}}},
+ {"/cmd/test/3", false, "/cmd/:tool/:sub", Params{Param{Key: "tool", Value: "test"}, Param{Key: "sub", Value: "3"}}},
+ {"/cmd/who", true, "/cmd/:tool/", Params{Param{"tool", "who"}}},
+ {"/cmd/who/", false, "/cmd/:tool/", Params{Param{"tool", "who"}}},
+ {"/cmd/whoami", false, "/cmd/whoami", nil},
+ {"/cmd/whoami/", true, "/cmd/whoami", nil},
+ {"/cmd/whoami/r", false, "/cmd/:tool/:sub", Params{Param{Key: "tool", Value: "whoami"}, Param{Key: "sub", Value: "r"}}},
+ {"/cmd/whoami/r/", true, "/cmd/:tool/:sub", Params{Param{Key: "tool", Value: "whoami"}, Param{Key: "sub", Value: "r"}}},
+ {"/cmd/whoami/root", false, "/cmd/whoami/root", nil},
+ {"/cmd/whoami/root/", false, "/cmd/whoami/root/", nil},
+ {"/src/", false, "/src/*filepath", Params{Param{Key: "filepath", Value: "/"}}},
+ {"/src/some/file.png", false, "/src/*filepath", Params{Param{Key: "filepath", Value: "/some/file.png"}}},
+ {"/search/", false, "/search/", nil},
+ {"/search/someth!ng+in+ünìcodé", false, "/search/:query", Params{Param{Key: "query", Value: "someth!ng+in+ünìcodé"}}},
+ {"/search/someth!ng+in+ünìcodé/", true, "", Params{Param{Key: "query", Value: "someth!ng+in+ünìcodé"}}},
+ {"/search/gin", false, "/search/:query", Params{Param{"query", "gin"}}},
+ {"/search/gin-gonic", false, "/search/gin-gonic", nil},
+ {"/search/google", false, "/search/google", nil},
+ {"/user_gopher", false, "/user_:name", Params{Param{Key: "name", Value: "gopher"}}},
+ {"/user_gopher/about", false, "/user_:name/about", Params{Param{Key: "name", Value: "gopher"}}},
+ {"/files/js/inc/framework.js", false, "/files/:dir/*filepath", Params{Param{Key: "dir", Value: "js"}, Param{Key: "filepath", Value: "/inc/framework.js"}}},
+ {"/info/gordon/public", false, "/info/:user/public", Params{Param{Key: "user", Value: "gordon"}}},
+ {"/info/gordon/project/go", false, "/info/:user/project/:project", Params{Param{Key: "user", Value: "gordon"}, Param{Key: "project", Value: "go"}}},
+ {"/info/gordon/project/golang", false, "/info/:user/project/golang", Params{Param{Key: "user", Value: "gordon"}}},
+ {"/aa/aa", false, "/aa/*xx", Params{Param{Key: "xx", Value: "/aa"}}},
+ {"/ab/ab", false, "/ab/*xx", Params{Param{Key: "xx", Value: "/ab"}}},
+ {"/a", false, "/:cc", Params{Param{Key: "cc", Value: "a"}}},
+ // * Error with argument being intercepted
+ // new PR handle (/all /all/cc /a/cc)
+ // fix PR: https://github.com/gin-gonic/gin/pull/2796
+ {"/all", false, "/:cc", Params{Param{Key: "cc", Value: "all"}}},
+ {"/d", false, "/:cc", Params{Param{Key: "cc", Value: "d"}}},
+ {"/ad", false, "/:cc", Params{Param{Key: "cc", Value: "ad"}}},
+ {"/dd", false, "/:cc", Params{Param{Key: "cc", Value: "dd"}}},
+ {"/dddaa", false, "/:cc", Params{Param{Key: "cc", Value: "dddaa"}}},
+ {"/aa", false, "/:cc", Params{Param{Key: "cc", Value: "aa"}}},
+ {"/aaa", false, "/:cc", Params{Param{Key: "cc", Value: "aaa"}}},
+ {"/aaa/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "aaa"}}},
+ {"/ab", false, "/:cc", Params{Param{Key: "cc", Value: "ab"}}},
+ {"/abb", false, "/:cc", Params{Param{Key: "cc", Value: "abb"}}},
+ {"/abb/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "abb"}}},
+ {"/allxxxx", false, "/:cc", Params{Param{Key: "cc", Value: "allxxxx"}}},
+ {"/alldd", false, "/:cc", Params{Param{Key: "cc", Value: "alldd"}}},
+ {"/all/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "all"}}},
+ {"/a/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "a"}}},
+ {"/cc/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "cc"}}},
+ {"/ccc/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "ccc"}}},
+ {"/deedwjfs/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "deedwjfs"}}},
+ {"/acllcc/cc", false, "/:cc/cc", Params{Param{Key: "cc", Value: "acllcc"}}},
+ {"/get/test/abc/", false, "/get/test/abc/", nil},
+ {"/get/te/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "te"}}},
+ {"/get/testaa/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "testaa"}}},
+ {"/get/xx/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "xx"}}},
+ {"/get/tt/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "tt"}}},
+ {"/get/a/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "a"}}},
+ {"/get/t/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "t"}}},
+ {"/get/aa/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "aa"}}},
+ {"/get/abas/abc/", false, "/get/:param/abc/", Params{Param{Key: "param", Value: "abas"}}},
+ {"/something/secondthing/test", false, "/something/secondthing/test", nil},
+ {"/something/abcdad/thirdthing", false, "/something/:paramname/thirdthing", Params{Param{Key: "paramname", Value: "abcdad"}}},
+ {"/something/secondthingaaaa/thirdthing", false, "/something/:paramname/thirdthing", Params{Param{Key: "paramname", Value: "secondthingaaaa"}}},
+ {"/something/se/thirdthing", false, "/something/:paramname/thirdthing", Params{Param{Key: "paramname", Value: "se"}}},
+ {"/something/s/thirdthing", false, "/something/:paramname/thirdthing", Params{Param{Key: "paramname", Value: "s"}}},
+ {"/c/d/ee", false, "/:cc/:dd/ee", Params{Param{Key: "cc", Value: "c"}, Param{Key: "dd", Value: "d"}}},
+ {"/c/d/e/ff", false, "/:cc/:dd/:ee/ff", Params{Param{Key: "cc", Value: "c"}, Param{Key: "dd", Value: "d"}, Param{Key: "ee", Value: "e"}}},
+ {"/c/d/e/f/gg", false, "/:cc/:dd/:ee/:ff/gg", Params{Param{Key: "cc", Value: "c"}, Param{Key: "dd", Value: "d"}, Param{Key: "ee", Value: "e"}, Param{Key: "ff", Value: "f"}}},
+ {"/c/d/e/f/g/hh", false, "/:cc/:dd/:ee/:ff/:gg/hh", Params{Param{Key: "cc", Value: "c"}, Param{Key: "dd", Value: "d"}, Param{Key: "ee", Value: "e"}, Param{Key: "ff", Value: "f"}, Param{Key: "gg", Value: "g"}}},
+ {"/cc/dd/ee/ff/gg/hh", false, "/:cc/:dd/:ee/:ff/:gg/hh", Params{Param{Key: "cc", Value: "cc"}, Param{Key: "dd", Value: "dd"}, Param{Key: "ee", Value: "ee"}, Param{Key: "ff", Value: "ff"}, Param{Key: "gg", Value: "gg"}}},
+ {"/get/abc", false, "/get/abc", nil},
+ {"/get/a", false, "/get/:param", Params{Param{Key: "param", Value: "a"}}},
+ {"/get/abz", false, "/get/:param", Params{Param{Key: "param", Value: "abz"}}},
+ {"/get/12a", false, "/get/:param", Params{Param{Key: "param", Value: "12a"}}},
+ {"/get/abcd", false, "/get/:param", Params{Param{Key: "param", Value: "abcd"}}},
+ {"/get/abc/123abc", false, "/get/abc/123abc", nil},
+ {"/get/abc/12", false, "/get/abc/:param", Params{Param{Key: "param", Value: "12"}}},
+ {"/get/abc/123ab", false, "/get/abc/:param", Params{Param{Key: "param", Value: "123ab"}}},
+ {"/get/abc/xyz", false, "/get/abc/:param", Params{Param{Key: "param", Value: "xyz"}}},
+ {"/get/abc/123abcddxx", false, "/get/abc/:param", Params{Param{Key: "param", Value: "123abcddxx"}}},
+ {"/get/abc/123abc/xxx8", false, "/get/abc/123abc/xxx8", nil},
+ {"/get/abc/123abc/x", false, "/get/abc/123abc/:param", Params{Param{Key: "param", Value: "x"}}},
+ {"/get/abc/123abc/xxx", false, "/get/abc/123abc/:param", Params{Param{Key: "param", Value: "xxx"}}},
+ {"/get/abc/123abc/abc", false, "/get/abc/123abc/:param", Params{Param{Key: "param", Value: "abc"}}},
+ {"/get/abc/123abc/xxx8xxas", false, "/get/abc/123abc/:param", Params{Param{Key: "param", Value: "xxx8xxas"}}},
+ {"/get/abc/123abc/xxx8/1234", false, "/get/abc/123abc/xxx8/1234", nil},
+ {"/get/abc/123abc/xxx8/1", false, "/get/abc/123abc/xxx8/:param", Params{Param{Key: "param", Value: "1"}}},
+ {"/get/abc/123abc/xxx8/123", false, "/get/abc/123abc/xxx8/:param", Params{Param{Key: "param", Value: "123"}}},
+ {"/get/abc/123abc/xxx8/78k", false, "/get/abc/123abc/xxx8/:param", Params{Param{Key: "param", Value: "78k"}}},
+ {"/get/abc/123abc/xxx8/1234xxxd", false, "/get/abc/123abc/xxx8/:param", Params{Param{Key: "param", Value: "1234xxxd"}}},
+ {"/get/abc/123abc/xxx8/1234/ffas", false, "/get/abc/123abc/xxx8/1234/ffas", nil},
+ {"/get/abc/123abc/xxx8/1234/f", false, "/get/abc/123abc/xxx8/1234/:param", Params{Param{Key: "param", Value: "f"}}},
+ {"/get/abc/123abc/xxx8/1234/ffa", false, "/get/abc/123abc/xxx8/1234/:param", Params{Param{Key: "param", Value: "ffa"}}},
+ {"/get/abc/123abc/xxx8/1234/kka", false, "/get/abc/123abc/xxx8/1234/:param", Params{Param{Key: "param", Value: "kka"}}},
+ {"/get/abc/123abc/xxx8/1234/ffas321", false, "/get/abc/123abc/xxx8/1234/:param", Params{Param{Key: "param", Value: "ffas321"}}},
+ {"/get/abc/123abc/xxx8/1234/kkdd/12c", false, "/get/abc/123abc/xxx8/1234/kkdd/12c", nil},
+ {"/get/abc/123abc/xxx8/1234/kkdd/1", false, "/get/abc/123abc/xxx8/1234/kkdd/:param", Params{Param{Key: "param", Value: "1"}}},
+ {"/get/abc/123abc/xxx8/1234/kkdd/12", false, "/get/abc/123abc/xxx8/1234/kkdd/:param", Params{Param{Key: "param", Value: "12"}}},
+ {"/get/abc/123abc/xxx8/1234/kkdd/12b", false, "/get/abc/123abc/xxx8/1234/kkdd/:param", Params{Param{Key: "param", Value: "12b"}}},
+ {"/get/abc/123abc/xxx8/1234/kkdd/34", false, "/get/abc/123abc/xxx8/1234/kkdd/:param", Params{Param{Key: "param", Value: "34"}}},
+ {"/get/abc/123abc/xxx8/1234/kkdd/12c2e3", false, "/get/abc/123abc/xxx8/1234/kkdd/:param", Params{Param{Key: "param", Value: "12c2e3"}}},
+ {"/get/abc/12/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "12"}}},
+ {"/get/abc/123abdd/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "123abdd"}}},
+ {"/get/abc/123abdddf/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "123abdddf"}}},
+ {"/get/abc/123ab/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "123ab"}}},
+ {"/get/abc/123abgg/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "123abgg"}}},
+ {"/get/abc/123abff/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "123abff"}}},
+ {"/get/abc/123abffff/test", false, "/get/abc/:param/test", Params{Param{Key: "param", Value: "123abffff"}}},
+ {"/get/abc/123abd/test", false, "/get/abc/123abd/:param", Params{Param{Key: "param", Value: "test"}}},
+ {"/get/abc/123abddd/test", false, "/get/abc/123abddd/:param", Params{Param{Key: "param", Value: "test"}}},
+ {"/get/abc/123/test22", false, "/get/abc/123/:param", Params{Param{Key: "param", Value: "test22"}}},
+ {"/get/abc/123abg/test", false, "/get/abc/123abg/:param", Params{Param{Key: "param", Value: "test"}}},
+ {"/get/abc/123abf/testss", false, "/get/abc/123abf/:param", Params{Param{Key: "param", Value: "testss"}}},
+ {"/get/abc/123abfff/te", false, "/get/abc/123abfff/:param", Params{Param{Key: "param", Value: "te"}}},
+ })
+
+ checkPriorities(t, tree)
+}
+
+func TestUnescapeParameters(t *testing.T) {
+ tree := &Node{}
+
+ routes := [...]string{
+ "/",
+ "/cmd/:tool/:sub",
+ "/cmd/:tool/",
+ "/src/*filepath",
+ "/search/:query",
+ "/files/:dir/*filepath",
+ "/info/:user/project/:project",
+ "/info/:user",
+ }
+ for _, route := range routes {
+ tree.addRoute(route, fakeHandler(route))
+ }
+
+ checkRequests(t, tree, testRequests{
+ {"/", false, "/", nil},
+ {"/cmd/test/", false, "/cmd/:tool/", Params{Param{Key: "tool", Value: "test"}}},
+ {"/cmd/test", true, "", Params{Param{Key: "tool", Value: "test"}}},
+ {"/src/some/file.png", false, "/src/*filepath", Params{Param{Key: "filepath", Value: "/some/file.png"}}},
+ {"/src/some/file+test.png", false, "/src/*filepath", Params{Param{Key: "filepath", Value: "/some/file test.png"}}},
+ {"/src/some/file++++%%%%test.png", false, "/src/*filepath", Params{Param{Key: "filepath", Value: "/some/file++++%%%%test.png"}}},
+ {"/src/some/file%2Ftest.png", false, "/src/*filepath", Params{Param{Key: "filepath", Value: "/some/file/test.png"}}},
+ {"/search/someth!ng+in+ünìcodé", false, "/search/:query", Params{Param{Key: "query", Value: "someth!ng in ünìcodé"}}},
+ {"/info/gordon/project/go", false, "/info/:user/project/:project", Params{Param{Key: "user", Value: "gordon"}, Param{Key: "project", Value: "go"}}},
+ {"/info/slash%2Fgordon", false, "/info/:user", Params{Param{Key: "user", Value: "slash/gordon"}}},
+ {"/info/slash%2Fgordon/project/Project%20%231", false, "/info/:user/project/:project", Params{Param{Key: "user", Value: "slash/gordon"}, Param{Key: "project", Value: "Project #1"}}},
+ {"/info/slash%%%%", false, "/info/:user", Params{Param{Key: "user", Value: "slash%%%%"}}},
+ {"/info/slash%%%%2Fgordon/project/Project%%%%20%231", false, "/info/:user/project/:project", Params{Param{Key: "user", Value: "slash%%%%2Fgordon"}, Param{Key: "project", Value: "Project%%%%20%231"}}},
+ }, true)
+
+ checkPriorities(t, tree)
+}
+
+func catchPanic(testFunc func()) (recv interface{}) {
+ defer func() {
+ recv = recover()
+ }()
+
+ testFunc()
+ return
+}
+
+type testRoute struct {
+ path string
+ conflict bool
+}
+
+func testRoutes(t *testing.T, routes []testRoute) {
+ tree := &Node{}
+
+ for _, route := range routes {
+ recv := catchPanic(func() {
+ tree.addRoute(route.path, nil)
+ })
+
+ if route.conflict {
+ if recv == nil {
+ t.Errorf("no panic for conflicting route '%s'", route.path)
+ }
+ } else if recv != nil {
+ t.Errorf("unexpected panic for route '%s': %v", route.path, recv)
+ }
+ }
+}
+
+func TestTreeWildcardConflict(t *testing.T) {
+ routes := []testRoute{
+ {"/cmd/:tool/:sub", false},
+ {"/cmd/vet", false},
+ {"/foo/bar", false},
+ {"/foo/:name", false},
+ {"/foo/:names", true},
+ {"/cmd/*path", true},
+ {"/cmd/:badvar", true},
+ {"/cmd/:tool/names", false},
+ {"/cmd/:tool/:badsub/details", true},
+ {"/src/*filepath", false},
+ {"/src/:file", true},
+ {"/src/static.json", true},
+ {"/src/*filepathx", true},
+ {"/src/", true},
+ {"/src/foo/bar", true},
+ {"/src1/", false},
+ {"/src1/*filepath", true},
+ {"/src2*filepath", true},
+ {"/src2/*filepath", false},
+ {"/search/:query", false},
+ {"/search/valid", false},
+ {"/user_:name", false},
+ {"/user_x", false},
+ {"/user_:name", false},
+ {"/id:id", false},
+ {"/id/:id", false},
+ }
+ testRoutes(t, routes)
+}
+
+func TestCatchAllAfterSlash(t *testing.T) {
+ routes := []testRoute{
+ {"/non-leading-*catchall", true},
+ }
+ testRoutes(t, routes)
+}
+
+func TestTreeChildConflict(t *testing.T) {
+ routes := []testRoute{
+ {"/cmd/vet", false},
+ {"/cmd/:tool", false},
+ {"/cmd/:tool/:sub", false},
+ {"/cmd/:tool/misc", false},
+ {"/cmd/:tool/:othersub", true},
+ {"/src/AUTHORS", false},
+ {"/src/*filepath", true},
+ {"/user_x", false},
+ {"/user_:name", false},
+ {"/id/:id", false},
+ {"/id:id", false},
+ {"/:id", false},
+ {"/*filepath", true},
+ }
+ testRoutes(t, routes)
+}
+
+func TestTreeDuplicatePath(t *testing.T) {
+ tree := &Node{}
+
+ routes := [...]string{
+ "/",
+ "/doc/",
+ "/src/*filepath",
+ "/search/:query",
+ "/user_:name",
+ }
+ for _, route := range routes {
+ recv := catchPanic(func() {
+ tree.addRoute(route, fakeHandler(route))
+ })
+ if recv != nil {
+ t.Fatalf("panic inserting route '%s': %v", route, recv)
+ }
+
+ // Add again
+ recv = catchPanic(func() {
+ tree.addRoute(route, nil)
+ })
+ if recv == nil {
+ t.Fatalf("no panic while inserting duplicate route '%s", route)
+ }
+ }
+
+ //printChildren(tree, "")
+
+ checkRequests(t, tree, testRequests{
+ {"/", false, "/", nil},
+ {"/doc/", false, "/doc/", nil},
+ {"/src/some/file.png", false, "/src/*filepath", Params{Param{"filepath", "/some/file.png"}}},
+ {"/search/someth!ng+in+ünìcodé", false, "/search/:query", Params{Param{"query", "someth!ng+in+ünìcodé"}}},
+ {"/user_gopher", false, "/user_:name", Params{Param{"name", "gopher"}}},
+ })
+}
+
+func TestEmptyWildcardName(t *testing.T) {
+ tree := &Node{}
+
+ routes := [...]string{
+ "/user:",
+ "/user:/",
+ "/cmd/:/",
+ "/src/*",
+ }
+ for _, route := range routes {
+ recv := catchPanic(func() {
+ tree.addRoute(route, nil)
+ })
+ if recv == nil {
+ t.Fatalf("no panic while inserting route with empty wildcard name '%s", route)
+ }
+ }
+}
+
+func TestTreeCatchAllConflict(t *testing.T) {
+ routes := []testRoute{
+ {"/src/*filepath/x", true},
+ {"/src2/", false},
+ {"/src2/*filepath/x", true},
+ {"/src3/*filepath", false},
+ {"/src3/*filepath/x", true},
+ }
+ testRoutes(t, routes)
+}
+
+func TestTreeCatchAllConflictRoot(t *testing.T) {
+ routes := []testRoute{
+ {"/", false},
+ {"/*filepath", true},
+ }
+ testRoutes(t, routes)
+}
+
+func TestTreeCatchMaxParams(t *testing.T) {
+ tree := &Node{}
+ var route = "/cmd/*filepath"
+ tree.addRoute(route, fakeHandler(route))
+}
+
+func TestTreeDoubleWildcard(t *testing.T) {
+ const panicMsg = "only one wildcard per path segment is allowed"
+
+ routes := [...]string{
+ "/:foo:bar",
+ "/:foo:bar/",
+ "/:foo*bar",
+ }
+
+ for _, route := range routes {
+ tree := &Node{}
+ recv := catchPanic(func() {
+ tree.addRoute(route, nil)
+ })
+
+ if rs, ok := recv.(string); !ok || !strings.HasPrefix(rs, panicMsg) {
+ t.Fatalf(`"Expected panic "%s" for route '%s', got "%v"`, panicMsg, route, recv)
+ }
+ }
+}
+
+/*func TestTreeDuplicateWildcard(t *testing.T) {
+ tree := &Node{}
+ routes := [...]string{
+ "/:id/:name/:id",
+ }
+ for _, route := range routes {
+ ...
+ }
+}*/
+
+func TestTreeTrailingSlashRedirect(t *testing.T) {
+ tree := &Node{}
+
+ routes := [...]string{
+ "/hi",
+ "/b/",
+ "/search/:query",
+ "/cmd/:tool/",
+ "/src/*filepath",
+ "/x",
+ "/x/y",
+ "/y/",
+ "/y/z",
+ "/0/:id",
+ "/0/:id/1",
+ "/1/:id/",
+ "/1/:id/2",
+ "/aa",
+ "/a/",
+ "/admin",
+ "/admin/:category",
+ "/admin/:category/:page",
+ "/doc",
+ "/doc/go_faq.html",
+ "/doc/go1.html",
+ "/no/a",
+ "/no/b",
+ "/api/hello/:name",
+ }
+ for _, route := range routes {
+ recv := catchPanic(func() {
+ tree.addRoute(route, fakeHandler(route))
+ })
+ if recv != nil {
+ t.Fatalf("panic inserting route '%s': %v", route, recv)
+ }
+ }
+
+ tsrRoutes := [...]string{
+ "/hi/",
+ "/b",
+ "/search/gopher/",
+ "/cmd/vet",
+ "/src",
+ "/x/",
+ "/y",
+ "/0/go/",
+ "/1/go",
+ "/a",
+ "/admin/",
+ "/admin/config/",
+ "/admin/config/permissions/",
+ "/doc/",
+ }
+ for _, route := range tsrRoutes {
+ value := tree.getValue(route, nil, false)
+ if value.Handler != nil {
+ t.Fatalf("non-nil handler for TSR route '%s", route)
+ } else if !value.Tsr {
+ t.Errorf("expected TSR recommendation for route '%s'", route)
+ }
+ }
+
+ noTsrRoutes := [...]string{
+ "/",
+ "/no",
+ "/no/",
+ "/_",
+ "/_/",
+ "/api/world/abc",
+ }
+ for _, route := range noTsrRoutes {
+ value := tree.getValue(route, nil, false)
+ if value.Handler != nil {
+ t.Fatalf("non-nil handler for No-TSR route '%s", route)
+ } else if value.Tsr {
+ t.Errorf("expected no TSR recommendation for route '%s'", route)
+ }
+ }
+}
+
+func TestTreeRootTrailingSlashRedirect(t *testing.T) {
+ tree := &Node{}
+
+ recv := catchPanic(func() {
+ tree.addRoute("/:test", fakeHandler("/:test"))
+ })
+ if recv != nil {
+ t.Fatalf("panic inserting test route: %v", recv)
+ }
+
+ value := tree.getValue("/", nil, false)
+ if value.Handler != nil {
+ t.Fatalf("non-nil handler")
+ } else if value.Tsr {
+ t.Errorf("expected no TSR recommendation")
+ }
+}
+
+func TestTreeFindCaseInsensitivePath(t *testing.T) {
+ tree := &Node{}
+
+ longPath := "/l" + strings.Repeat("o", 128) + "ng"
+ lOngPath := "/l" + strings.Repeat("O", 128) + "ng/"
+
+ routes := [...]string{
+ "/hi",
+ "/b/",
+ "/ABC/",
+ "/search/:query",
+ "/cmd/:tool/",
+ "/src/*filepath",
+ "/x",
+ "/x/y",
+ "/y/",
+ "/y/z",
+ "/0/:id",
+ "/0/:id/1",
+ "/1/:id/",
+ "/1/:id/2",
+ "/aa",
+ "/a/",
+ "/doc",
+ "/doc/go_faq.html",
+ "/doc/go1.html",
+ "/doc/go/away",
+ "/no/a",
+ "/no/b",
+ "/Π",
+ "/u/apfêl/",
+ "/u/äpfêl/",
+ "/u/öpfêl",
+ "/v/Äpfêl/",
+ "/v/Öpfêl",
+ "/w/♬", // 3 byte
+ "/w/♭/", // 3 byte, last byte differs
+ "/w/𠜎", // 4 byte
+ "/w/𠜏/", // 4 byte
+ longPath,
+ }
+
+ for _, route := range routes {
+ recv := catchPanic(func() {
+ tree.addRoute(route, fakeHandler(route))
+ })
+ if recv != nil {
+ t.Fatalf("panic inserting route '%s': %v", route, recv)
+ }
+ }
+
+ // Check out == in for all registered routes
+ // With fixTrailingSlash = true
+ for _, route := range routes {
+ out, found := tree.findCaseInsensitivePath(route, true)
+ if !found {
+ t.Errorf("Route '%s' not found!", route)
+ } else if string(out) != route {
+ t.Errorf("Wrong result for route '%s': %s", route, string(out))
+ }
+ }
+ // With fixTrailingSlash = false
+ for _, route := range routes {
+ out, found := tree.findCaseInsensitivePath(route, false)
+ if !found {
+ t.Errorf("Route '%s' not found!", route)
+ } else if string(out) != route {
+ t.Errorf("Wrong result for route '%s': %s", route, string(out))
+ }
+ }
+
+ tests := []struct {
+ in string
+ out string
+ found bool
+ slash bool
+ }{
+ {"/HI", "/hi", true, false},
+ {"/HI/", "/hi", true, true},
+ {"/B", "/b/", true, true},
+ {"/B/", "/b/", true, false},
+ {"/abc", "/ABC/", true, true},
+ {"/abc/", "/ABC/", true, false},
+ {"/aBc", "/ABC/", true, true},
+ {"/aBc/", "/ABC/", true, false},
+ {"/abC", "/ABC/", true, true},
+ {"/abC/", "/ABC/", true, false},
+ {"/SEARCH/QUERY", "/search/QUERY", true, false},
+ {"/SEARCH/QUERY/", "/search/QUERY", true, true},
+ {"/CMD/TOOL/", "/cmd/TOOL/", true, false},
+ {"/CMD/TOOL", "/cmd/TOOL/", true, true},
+ {"/SRC/FILE/PATH", "/src/FILE/PATH", true, false},
+ {"/x/Y", "/x/y", true, false},
+ {"/x/Y/", "/x/y", true, true},
+ {"/X/y", "/x/y", true, false},
+ {"/X/y/", "/x/y", true, true},
+ {"/X/Y", "/x/y", true, false},
+ {"/X/Y/", "/x/y", true, true},
+ {"/Y/", "/y/", true, false},
+ {"/Y", "/y/", true, true},
+ {"/Y/z", "/y/z", true, false},
+ {"/Y/z/", "/y/z", true, true},
+ {"/Y/Z", "/y/z", true, false},
+ {"/Y/Z/", "/y/z", true, true},
+ {"/y/Z", "/y/z", true, false},
+ {"/y/Z/", "/y/z", true, true},
+ {"/Aa", "/aa", true, false},
+ {"/Aa/", "/aa", true, true},
+ {"/AA", "/aa", true, false},
+ {"/AA/", "/aa", true, true},
+ {"/aA", "/aa", true, false},
+ {"/aA/", "/aa", true, true},
+ {"/A/", "/a/", true, false},
+ {"/A", "/a/", true, true},
+ {"/DOC", "/doc", true, false},
+ {"/DOC/", "/doc", true, true},
+ {"/NO", "", false, true},
+ {"/DOC/GO", "", false, true},
+ {"/π", "/Π", true, false},
+ {"/π/", "/Π", true, true},
+ {"/u/ÄPFÊL/", "/u/äpfêl/", true, false},
+ {"/u/ÄPFÊL", "/u/äpfêl/", true, true},
+ {"/u/ÖPFÊL/", "/u/öpfêl", true, true},
+ {"/u/ÖPFÊL", "/u/öpfêl", true, false},
+ {"/v/äpfêL/", "/v/Äpfêl/", true, false},
+ {"/v/äpfêL", "/v/Äpfêl/", true, true},
+ {"/v/öpfêL/", "/v/Öpfêl", true, true},
+ {"/v/öpfêL", "/v/Öpfêl", true, false},
+ {"/w/♬/", "/w/♬", true, true},
+ {"/w/♭", "/w/♭/", true, true},
+ {"/w/𠜎/", "/w/𠜎", true, true},
+ {"/w/𠜏", "/w/𠜏/", true, true},
+ {lOngPath, longPath, true, true},
+ }
+ // With fixTrailingSlash = true
+ for _, test := range tests {
+ out, found := tree.findCaseInsensitivePath(test.in, true)
+ if found != test.found || (found && (string(out) != test.out)) {
+ t.Errorf("Wrong result for '%s': got %s, %t; want %s, %t",
+ test.in, string(out), found, test.out, test.found)
+ return
+ }
+ }
+ // With fixTrailingSlash = false
+ for _, test := range tests {
+ out, found := tree.findCaseInsensitivePath(test.in, false)
+ if test.slash {
+ if found { // test needs a trailingSlash fix. It must not be found!
+ t.Errorf("Found without fixTrailingSlash: %s; got %s", test.in, string(out))
+ }
+ } else {
+ if found != test.found || (found && (string(out) != test.out)) {
+ t.Errorf("Wrong result for '%s': got %s, %t; want %s, %t",
+ test.in, string(out), found, test.out, test.found)
+ return
+ }
+ }
+ }
+}
+
+func TestTreeInvalidNodeType(t *testing.T) {
+ const panicMsg = "invalid Node type"
+
+ tree := &Node{}
+ tree.addRoute("/", fakeHandler("/"))
+ tree.addRoute("/:page", fakeHandler("/:page"))
+
+ // set invalid Node type
+ tree.children[0].nType = 42
+
+ // normal lookup
+ recv := catchPanic(func() {
+ tree.getValue("/test", nil, false)
+ })
+ if rs, ok := recv.(string); !ok || rs != panicMsg {
+ t.Fatalf("Expected panic '"+panicMsg+"', got '%v'", recv)
+ }
+
+ // case-insensitive lookup
+ recv = catchPanic(func() {
+ tree.findCaseInsensitivePath("/test", true)
+ })
+ if rs, ok := recv.(string); !ok || rs != panicMsg {
+ t.Fatalf("Expected panic '"+panicMsg+"', got '%v'", recv)
+ }
+}
+
+func TestTreeInvalidParamsType(t *testing.T) {
+ tree := &Node{}
+ tree.wildChild = true
+ tree.children = append(tree.children, &Node{})
+ tree.children[0].nType = 2
+
+ // set invalid Params type
+ params := make(Params, 0)
+
+ // try to trigger slice bounds out of range with capacity 0
+ tree.getValue("/test", ¶ms, false)
+}
+
+func TestTreeWildcardConflictEx(t *testing.T) {
+ conflicts := [...]struct {
+ route string
+ segPath string
+ existPath string
+ existSegPath string
+ }{
+ {"/who/are/foo", "/foo", `/who/are/\*you`, `/\*you`},
+ {"/who/are/foo/", "/foo/", `/who/are/\*you`, `/\*you`},
+ {"/who/are/foo/bar", "/foo/bar", `/who/are/\*you`, `/\*you`},
+ {"/con:nection", ":nection", `/con:tact`, `:tact`},
+ }
+
+ for _, conflict := range conflicts {
+ // I have to re-create a 'tree', because the 'tree' will be
+ // in an inconsistent state when the loop recovers from the
+ // panic which threw by 'addRoute' function.
+ tree := &Node{}
+ routes := [...]string{
+ "/con:tact",
+ "/who/are/*you",
+ "/who/foo/hello",
+ }
+
+ for _, route := range routes {
+ tree.addRoute(route, fakeHandler(route))
+ }
+
+ recv := catchPanic(func() {
+ tree.addRoute(conflict.route, fakeHandler(conflict.route))
+ })
+
+ if !regexp.MustCompile(fmt.Sprintf("'%s' in new path .* conflicts with existing wildcard '%s' in existing prefix '%s'", conflict.segPath, conflict.existSegPath, conflict.existPath)).MatchString(fmt.Sprint(recv)) {
+ t.Fatalf("invalid wildcard conflict error (%v)", recv)
+ }
+ }
+}
diff --git a/pkg/services/live/pushhttp/push.go b/pkg/services/live/pushhttp/push.go
index c8d0594b865..c3e5017aeb6 100644
--- a/pkg/services/live/pushhttp/push.go
+++ b/pkg/services/live/pushhttp/push.go
@@ -12,6 +12,8 @@ import (
"github.com/grafana/grafana/pkg/services/live/convert"
"github.com/grafana/grafana/pkg/services/live/pushurl"
"github.com/grafana/grafana/pkg/setting"
+
+ liveDto "github.com/grafana/grafana-plugin-sdk-go/live"
)
var (
@@ -45,7 +47,7 @@ func (g *Gateway) Run(ctx context.Context) error {
func (g *Gateway) Handle(ctx *models.ReqContext) {
streamID := ctx.Params(":streamId")
- stream, err := g.GrafanaLive.ManagedStreamRunner.GetOrCreateStream(ctx.SignedInUser.OrgId, streamID)
+ stream, err := g.GrafanaLive.ManagedStreamRunner.GetOrCreateStream(ctx.SignedInUser.OrgId, liveDto.ScopeStream, streamID)
if err != nil {
logger.Error("Error getting stream", "error", err)
ctx.Resp.WriteHeader(http.StatusInternalServerError)
@@ -92,3 +94,39 @@ func (g *Gateway) Handle(ctx *models.ReqContext) {
}
}
}
+
+func (g *Gateway) HandlePath(ctx *models.ReqContext) {
+ streamID := ctx.Params(":streamId")
+ path := ctx.Params(":path")
+
+ body, err := io.ReadAll(ctx.Req.Body)
+ if err != nil {
+ logger.Error("Error reading body", "error", err)
+ ctx.Resp.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ logger.Debug("Live channel push request",
+ "protocol", "http",
+ "streamId", streamID,
+ "path", path,
+ "bodyLength", len(body),
+ )
+
+ channelID := "stream/" + streamID + "/" + path
+
+ ruleFound, err := g.GrafanaLive.Pipeline.ProcessInput(ctx.Req.Context(), ctx.OrgId, channelID, body)
+ if err != nil {
+ logger.Error("Pipeline input processing error", "error", err, "body", string(body))
+ if errors.Is(err, liveDto.ErrInvalidChannelID) {
+ ctx.Resp.WriteHeader(http.StatusBadRequest)
+ } else {
+ ctx.Resp.WriteHeader(http.StatusInternalServerError)
+ }
+ return
+ }
+ if !ruleFound {
+ logger.Error("No conversion rule for a channel", "error", err, "channel", channelID)
+ ctx.Resp.WriteHeader(http.StatusNotFound)
+ return
+ }
+}
diff --git a/pkg/services/live/pushws/push.go b/pkg/services/live/pushws/push.go
index 1f513633c04..81a68f6101d 100644
--- a/pkg/services/live/pushws/push.go
+++ b/pkg/services/live/pushws/push.go
@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/services/live/pushurl"
"github.com/gorilla/websocket"
+ liveDto "github.com/grafana/grafana-plugin-sdk-go/live"
)
var (
@@ -165,7 +166,7 @@ func (s *Handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
break
}
- stream, err := s.managedStreamRunner.GetOrCreateStream(user.OrgId, streamID)
+ stream, err := s.managedStreamRunner.GetOrCreateStream(user.OrgId, liveDto.ScopeStream, streamID)
if err != nil {
logger.Error("Error getting stream", "error", err)
continue
diff --git a/pkg/services/live/remotewrite/convert.go b/pkg/services/live/remotewrite/convert.go
new file mode 100644
index 00000000000..a1c4e7fabfe
--- /dev/null
+++ b/pkg/services/live/remotewrite/convert.go
@@ -0,0 +1,139 @@
+package remotewrite
+
+import (
+ "strings"
+ "unicode"
+)
+
+type table struct {
+ First *unicode.RangeTable
+ Rest *unicode.RangeTable
+}
+
+var metricNameTable = table{
+ First: &unicode.RangeTable{
+ R16: []unicode.Range16{
+ {0x003A, 0x003A, 1}, // :
+ {0x0041, 0x005A, 1}, // A-Z
+ {0x005F, 0x005F, 1}, // _
+ {0x0061, 0x007A, 1}, // a-z
+ },
+ LatinOffset: 4,
+ },
+ Rest: &unicode.RangeTable{
+ R16: []unicode.Range16{
+ {0x0030, 0x003A, 1}, // 0-:
+ {0x0041, 0x005A, 1}, // A-Z
+ {0x005F, 0x005F, 1}, // _
+ {0x0061, 0x007A, 1}, // a-z
+ },
+ LatinOffset: 4,
+ },
+}
+
+var labelNameTable = table{
+ First: &unicode.RangeTable{
+ R16: []unicode.Range16{
+ {0x0041, 0x005A, 1}, // A-Z
+ {0x005F, 0x005F, 1}, // _
+ {0x0061, 0x007A, 1}, // a-z
+ },
+ LatinOffset: 3,
+ },
+ Rest: &unicode.RangeTable{
+ R16: []unicode.Range16{
+ {0x0030, 0x0039, 1}, // 0-9
+ {0x0041, 0x005A, 1}, // A-Z
+ {0x005F, 0x005F, 1}, // _
+ {0x0061, 0x007A, 1}, // a-z
+ },
+ LatinOffset: 4,
+ },
+}
+
+func isValid(name string, table table) bool {
+ if name == "" {
+ return false
+ }
+
+ for i, r := range name {
+ switch {
+ case i == 0:
+ if !unicode.In(r, table.First) {
+ return false
+ }
+ default:
+ if !unicode.In(r, table.Rest) {
+ return false
+ }
+ }
+ }
+
+ return true
+}
+
+// Sanitize checks if the name is valid according to the table. If not, it
+// attempts to replaces invalid runes with an underscore to create a valid
+// name.
+func sanitize(name string, table table) (string, bool) {
+ if isValid(name, table) {
+ return name, true
+ }
+
+ var b strings.Builder
+
+ for i, r := range name {
+ switch {
+ case i == 0:
+ if unicode.In(r, table.First) {
+ b.WriteRune(r)
+ }
+ default:
+ if unicode.In(r, table.Rest) {
+ b.WriteRune(r)
+ } else {
+ b.WriteString("_")
+ }
+ }
+ }
+
+ name = strings.Trim(b.String(), "_")
+ if name == "" {
+ return "", false
+ }
+
+ return name, true
+}
+
+// sanitizeMetricName checks if the name is a valid Prometheus metric name. If
+// not, it attempts to replaces invalid runes with an underscore to create a
+// valid name.
+func sanitizeMetricName(name string) (string, bool) {
+ return sanitize(name, metricNameTable)
+}
+
+// sanitizeLabelName checks if the name is a valid Prometheus label name. If
+// not, it attempts to replaces invalid runes with an underscore to create a
+// valid name.
+func sanitizeLabelName(name string) (string, bool) {
+ return sanitize(name, labelNameTable)
+}
+
+// sampleValue converts a field value into a value suitable for a simple sample value.
+func sampleValue(value interface{}) (float64, bool) {
+ switch v := value.(type) {
+ case float64:
+ return v, true
+ case int64:
+ return float64(v), true
+ case uint64:
+ return float64(v), true
+ case bool:
+ if v {
+ return 1.0, true
+ }
+ return 0.0, true
+ default:
+ return 0, false
+ }
+}
diff --git a/pkg/services/live/remotewrite/remotewrite.go b/pkg/services/live/remotewrite/remotewrite.go
new file mode 100644
index 00000000000..f8868eeb8c8
--- /dev/null
+++ b/pkg/services/live/remotewrite/remotewrite.go
@@ -0,0 +1,243 @@
+package remotewrite
+
+import (
+ "fmt"
+ "hash/fnv"
+ "strings"
+ "time"
+
+ "github.com/gogo/protobuf/proto"
+ "github.com/golang/snappy"
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/prometheus/prometheus/prompb"
+)
+
+type metricKey uint64
+
+// Serialize frames to Prometheus remote write format.
+func Serialize(frames ...*data.Frame) ([]byte, error) {
+ return TimeSeriesToBytes(TimeSeriesFromFrames(frames...))
+}
+
+// SerializeLabelsColumn frames to Prometheus remote write format.
+func SerializeLabelsColumn(frames ...*data.Frame) ([]byte, error) {
+ return TimeSeriesToBytes(TimeSeriesFromFramesLabelsColumn(frames...))
+}
+
+// TimeSeriesFromFrames converts frames to slice of Prometheus TimeSeries.
+func TimeSeriesFromFrames(frames ...*data.Frame) []prompb.TimeSeries {
+ var entries = make(map[metricKey]prompb.TimeSeries)
+ var keys []metricKey // sorted keys.
+
+ for _, frame := range frames {
+ timeFieldIndex, ok := timeFieldIndex(frame)
+ if !ok {
+ // Skipping frames without time field.
+ continue
+ }
+ for _, field := range frame.Fields {
+ if !field.Type().Numeric() {
+ continue
+ }
+ metricName := makeMetricName(frame, field)
+ metricName, ok := sanitizeMetricName(metricName)
+ if !ok {
+ continue
+ }
+
+ var samples []prompb.Sample
+
+ labels := createLabels(field.Labels)
+ key := makeMetricKey(metricName, labels)
+
+ for i := 0; i < field.Len(); i++ {
+ val, ok := field.ConcreteAt(i)
+ if !ok {
+ continue
+ }
+ value, ok := sampleValue(val)
+ if !ok {
+ continue
+ }
+ tm, ok := frame.Fields[timeFieldIndex].ConcreteAt(i)
+ if !ok {
+ continue
+ }
+ sample := prompb.Sample{
+ // Timestamp is int milliseconds for remote write.
+ Timestamp: toSampleTime(tm.(time.Time)),
+ Value: value,
+ }
+ samples = append(samples, sample)
+ }
+
+ labelsCopy := make([]prompb.Label, len(labels), len(labels)+1)
+ copy(labelsCopy, labels)
+ labelsCopy = append(labelsCopy, prompb.Label{
+ Name: "__name__",
+ Value: metricName,
+ })
+ promTimeSeries := prompb.TimeSeries{Labels: labelsCopy, Samples: samples}
+ entries[key] = promTimeSeries
+ keys = append(keys, key)
+ }
+ }
+
+ var promTimeSeriesBatch = make([]prompb.TimeSeries, 0, len(entries))
+ for _, key := range keys {
+ promTimeSeriesBatch = append(promTimeSeriesBatch, entries[key])
+ }
+
+ return promTimeSeriesBatch
+}
+
+// TimeSeriesFromFramesLabelsColumn converts frames to slice of Prometheus TimeSeries.
+func TimeSeriesFromFramesLabelsColumn(frames ...*data.Frame) []prompb.TimeSeries {
+ var entries = make(map[metricKey]prompb.TimeSeries)
+ var keys []metricKey // sorted keys.
+
+ for _, frame := range frames {
+ timeFieldIndex, ok := timeFieldIndex(frame)
+ if !ok {
+ // Skipping frames without time field.
+ continue
+ }
+
+ // Labels column frames have first column called "labels".
+ isLabelsColumnFrame := frame.Fields[0].Type() == data.FieldTypeString && frame.Fields[0].Name == "labels"
+
+ var labels [][]prompb.Label
+
+ if isLabelsColumnFrame {
+ labelsField := frame.Fields[0]
+ labels = make([][]prompb.Label, labelsField.Len())
+ for i := 0; i < labelsField.Len(); i++ {
+ val, ok := labelsField.ConcreteAt(i)
+ if !ok {
+ continue
+ }
+ parts := strings.Split(val.(string), ", ")
+ promLabels := make([]prompb.Label, 0)
+ for _, part := range parts {
+ labelParts := strings.SplitN(part, "=", 2)
+ if len(labelParts) != 2 {
+ continue
+ }
+ promLabels = append(promLabels, prompb.Label{Name: labelParts[0], Value: labelParts[1]})
+ }
+ labels[i] = promLabels
+ }
+ }
+
+ for _, field := range frame.Fields {
+ if !field.Type().Numeric() {
+ continue
+ }
+ metricName := makeMetricName(frame, field)
+ metricName, ok := sanitizeMetricName(metricName)
+ if !ok {
+ continue
+ }
+
+ for i := 0; i < field.Len(); i++ {
+ var labelsCopy []prompb.Label
+ if isLabelsColumnFrame && labels != nil {
+ labelsCopy = make([]prompb.Label, len(labels[i]), len(labels[i])+1)
+ copy(labelsCopy, labels[i])
+ } else {
+ labelsCopy = make([]prompb.Label, 0, len(field.Labels)+1)
+ for k, v := range field.Labels {
+ labelsCopy = append(labelsCopy, prompb.Label{Name: k, Value: v})
+ }
+ }
+
+ val, ok := field.ConcreteAt(i)
+ if !ok {
+ continue
+ }
+ value, ok := sampleValue(val)
+ if !ok {
+ continue
+ }
+ tm, ok := frame.Fields[timeFieldIndex].ConcreteAt(i)
+ if !ok {
+ continue
+ }
+ sample := prompb.Sample{
+ // Timestamp is int milliseconds for remote write.
+ Timestamp: toSampleTime(tm.(time.Time)),
+ Value: value,
+ }
+
+ labelsCopy = append(labelsCopy, prompb.Label{
+ Name: "__name__",
+ Value: metricName,
+ })
+ key := makeMetricKey(metricName, labelsCopy)
+
+ promTimeSeries := prompb.TimeSeries{Labels: labelsCopy, Samples: []prompb.Sample{sample}}
+ entries[key] = promTimeSeries
+ keys = append(keys, key)
+ }
+ }
+ }
+
+ var promTimeSeriesBatch = make([]prompb.TimeSeries, 0, len(entries))
+ for _, key := range keys {
+ promTimeSeriesBatch = append(promTimeSeriesBatch, entries[key])
+ }
+
+ return promTimeSeriesBatch
+}
+
+func timeFieldIndex(frame *data.Frame) (int, bool) {
+ timeFieldIndex := -1
+ for i, field := range frame.Fields {
+ if field.Type().Time() {
+ timeFieldIndex = i
+ break
+ }
+ }
+ return timeFieldIndex, timeFieldIndex > -1
+}
+
+func makeMetricName(frame *data.Frame, field *data.Field) string {
+ return frame.Name + "_" + field.Name
+}
+
+func toSampleTime(tm time.Time) int64 {
+ return tm.UnixNano() / int64(time.Millisecond)
+}
+
+// TimeSeriesToBytes converts Prometheus TimeSeries to snappy compressed byte slice.
+func TimeSeriesToBytes(ts []prompb.TimeSeries) ([]byte, error) {
+ writeRequestData, err := proto.Marshal(&prompb.WriteRequest{Timeseries: ts})
+ if err != nil {
+ return nil, fmt.Errorf("unable to marshal protobuf: %v", err)
+ }
+ return snappy.Encode(nil, writeRequestData), nil
+}
+
+func makeMetricKey(name string, labels []prompb.Label) metricKey {
+ h := fnv.New64a()
+ _, _ = h.Write([]byte(name))
+ for _, label := range labels {
+ _, _ = h.Write([]byte(label.Name))
+ _, _ = h.Write([]byte("\x00"))
+ _, _ = h.Write([]byte(label.Value))
+ _, _ = h.Write([]byte("\x00"))
+ }
+ return metricKey(h.Sum64())
+}
+
+func createLabels(fieldLabels map[string]string) []prompb.Label {
+ labels := make([]prompb.Label, 0, len(fieldLabels))
+ for k, v := range fieldLabels {
+ sanitizedName, ok := sanitizeLabelName(k)
+ if !ok {
+ continue
+ }
+ labels = append(labels, prompb.Label{Name: sanitizedName, Value: v})
+ }
+ return labels
+}
diff --git a/pkg/services/ngalert/api/Makefile b/pkg/services/ngalert/api/Makefile
index c84ba1c7ebe..68828e51b6c 100644
--- a/pkg/services/ngalert/api/Makefile
+++ b/pkg/services/ngalert/api/Makefile
@@ -19,6 +19,7 @@ copy-files:
fix:
sed -i -e 's/apimodels\.\[\]PostableAlert/apimodels.PostableAlerts/' $(GENERATED_GO_MATCHERS)
sed -i -e 's/apimodels\.\[\]UpdateDashboardAclCommand/apimodels.Permissions/' $(GENERATED_GO_MATCHERS)
+ sed -i -e 's/apimodels\.\[\]PostableApiReceiver/apimodels.TestReceiversConfigParams/' $(GENERATED_GO_MATCHERS)
goimports -w -v $(GENERATED_GO_MATCHERS)
clean:
diff --git a/pkg/services/ngalert/api/api_testing.go b/pkg/services/ngalert/api/api_testing.go
index a55bebd0853..f56d296b500 100644
--- a/pkg/services/ngalert/api/api_testing.go
+++ b/pkg/services/ngalert/api/api_testing.go
@@ -15,7 +15,6 @@ import (
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
- "github.com/grafana/grafana/pkg/util"
)
type TestingApiSrv struct {
@@ -26,11 +25,6 @@ type TestingApiSrv struct {
log log.Logger
}
-func (srv TestingApiSrv) RouteTestReceiverConfig(c *models.ReqContext, body apimodels.ExtendedReceiver) response.Response {
- srv.log.Info("RouteTestReceiverConfig: ", "body", body)
- return response.JSON(http.StatusOK, util.DynMap{"message": "success"})
-}
-
func (srv TestingApiSrv) RouteTestRuleConfig(c *models.ReqContext, body apimodels.TestRulePayload) response.Response {
recipient := c.Params("Recipient")
if recipient == apimodels.GrafanaBackend.String() {
diff --git a/pkg/services/ngalert/api/generated_base_api_testing.go b/pkg/services/ngalert/api/generated_base_api_testing.go
index 03aee5611f4..bf4dc80c138 100644
--- a/pkg/services/ngalert/api/generated_base_api_testing.go
+++ b/pkg/services/ngalert/api/generated_base_api_testing.go
@@ -21,7 +21,6 @@ import (
type TestingApiService interface {
RouteEvalQueries(*models.ReqContext, apimodels.EvalQueriesPayload) response.Response
- RouteTestReceiverConfig(*models.ReqContext, apimodels.ExtendedReceiver) response.Response
RouteTestRuleConfig(*models.ReqContext, apimodels.TestRulePayload) response.Response
}
@@ -37,16 +36,6 @@ func (api *API) RegisterTestingApiEndpoints(srv TestingApiService, m *metrics.Me
m,
),
)
- group.Post(
- toMacaronPath("/api/v1/receiver/test/{Recipient}"),
- binding.Bind(apimodels.ExtendedReceiver{}),
- metrics.Instrument(
- http.MethodPost,
- "/api/v1/receiver/test/{Recipient}",
- srv.RouteTestReceiverConfig,
- m,
- ),
- )
group.Post(
toMacaronPath("/api/v1/rule/test/{Recipient}"),
binding.Bind(apimodels.TestRulePayload{}),
diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go
index 74fa1491263..70490733096 100644
--- a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go
+++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go
@@ -83,7 +83,10 @@ import (
// 200: Ack
// 207: MultiStatus
// 400: ValidationError
+// 403: PermissionDenied
+// 404: AlertManagerNotFound
// 408: Failure
+// 409: AlertManagerNotReady
// swagger:route GET /api/alertmanager/{Recipient}/api/v2/silences alertmanager RouteGetSilences
//
@@ -118,12 +121,20 @@ import (
// 400: ValidationError
// swagger:model
-type TestReceiversConfig struct {
- Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
-}
+type PermissionDenied struct{}
+
+// swagger:model
+type AlertManagerNotFound struct{}
+
+// swagger:model
+type AlertManagerNotReady struct{}
+
+// swagger:model
+type MultiStatus struct{}
// swagger:parameters RoutePostTestReceivers
type TestReceiversConfigParams struct {
+ // in:body
Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
}
@@ -311,7 +322,7 @@ type BodyAlertingConfig struct {
}
// alertmanager routes
-// swagger:parameters RoutePostAlertingConfig RouteGetAlertingConfig RouteDeleteAlertingConfig RouteGetAMStatus RouteGetAMAlerts RoutePostAMAlerts RouteGetAMAlertGroups RouteGetSilences RouteCreateSilence RouteGetSilence RouteDeleteSilence RoutePostAlertingConfig
+// swagger:parameters RoutePostAlertingConfig RouteGetAlertingConfig RouteDeleteAlertingConfig RouteGetAMStatus RouteGetAMAlerts RoutePostAMAlerts RouteGetAMAlertGroups RouteGetSilences RouteCreateSilence RouteGetSilence RouteDeleteSilence RoutePostAlertingConfig RoutePostTestReceivers
// ruler routes
// swagger:parameters RouteGetRulesConfig RoutePostNameRulesConfig RouteGetNamespaceRulesConfig RouteDeleteNamespaceRulesConfig RouteGetRulegGroupConfig RouteDeleteRuleGroupConfig
// prom routes
diff --git a/pkg/services/ngalert/api/tooling/definitions/testing.go b/pkg/services/ngalert/api/tooling/definitions/testing.go
index adc2d52ee54..4e01fbf3ec1 100644
--- a/pkg/services/ngalert/api/tooling/definitions/testing.go
+++ b/pkg/services/ngalert/api/tooling/definitions/testing.go
@@ -12,21 +12,6 @@ import (
"github.com/prometheus/prometheus/promql"
)
-// swagger:route Post /api/v1/receiver/test/{Recipient} testing RouteTestReceiverConfig
-//
-// Test receiver
-//
-// Consumes:
-// - application/json
-//
-// Produces:
-// - application/json
-//
-// Responses:
-// 200: Success
-// 412: SmtpNotEnabled
-// 500: Failure
-
// swagger:route Post /api/v1/rule/test/{Recipient} testing RouteTestRuleConfig
//
// Test rule
diff --git a/pkg/services/ngalert/api/tooling/post.json b/pkg/services/ngalert/api/tooling/post.json
index a2a6aee0e6a..6cc9506a9e4 100644
--- a/pkg/services/ngalert/api/tooling/post.json
+++ b/pkg/services/ngalert/api/tooling/post.json
@@ -86,6 +86,14 @@
"type": "object",
"x-go-package": "github.com/prometheus/client_golang/api/prometheus/v1"
},
+ "AlertManagerNotFound": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
+ "AlertManagerNotReady": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
"AlertManagersResult": {
"properties": {
"activeAlertManagers": {
@@ -1095,6 +1103,10 @@
},
"type": "array"
},
+ "MultiStatus": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
"NamespaceConfigResponse": {
"additionalProperties": {
"items": {
@@ -1340,6 +1352,10 @@
"type": "object",
"x-go-package": "github.com/prometheus/alertmanager/config"
},
+ "PermissionDenied": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
"Point": {
"properties": {
"T": {
@@ -2286,19 +2302,6 @@
"type": "object",
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
},
- "TestReceiversConfig": {
- "properties": {
- "receivers": {
- "items": {
- "$ref": "#/definitions/PostableApiReceiver"
- },
- "type": "array",
- "x-go-name": "Receivers"
- }
- },
- "type": "object",
- "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
- },
"TestReceiversResult": {
"properties": {
"notified_at": {
@@ -2344,7 +2347,6 @@
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
},
"URL": {
- "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -2377,9 +2379,9 @@
"$ref": "#/definitions/Userinfo"
}
},
- "title": "A URL represents a parsed URL (technically, a URI reference).",
+ "title": "URL is a custom URL type that allows validation at configuration load time.",
"type": "object",
- "x-go-package": "net/url"
+ "x-go-package": "github.com/prometheus/common/config"
},
"Userinfo": {
"description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -2546,6 +2548,7 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"alertGroup": {
+ "description": "AlertGroup alert group",
"properties": {
"alerts": {
"description": "alerts",
@@ -2567,17 +2570,14 @@
"labels",
"receiver"
],
- "type": "object",
- "x-go-name": "AlertGroup",
- "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+ "type": "object"
},
"alertGroups": {
+ "description": "AlertGroups alert groups",
"items": {
"$ref": "#/definitions/alertGroup"
},
- "type": "array",
- "x-go-name": "AlertGroups",
- "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+ "type": "array"
},
"alertStatus": {
"description": "AlertStatus alert status",
@@ -2697,7 +2697,6 @@
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
- "description": "GettableAlert gettable alert",
"properties": {
"annotations": {
"$ref": "#/definitions/labelSet"
@@ -2756,7 +2755,9 @@
"status",
"updatedAt"
],
- "type": "object"
+ "type": "object",
+ "x-go-name": "GettableAlert",
+ "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"gettableAlerts": {
"items": {
@@ -2767,7 +2768,6 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"gettableSilence": {
- "description": "GettableSilence gettable silence",
"properties": {
"comment": {
"description": "comment",
@@ -2819,7 +2819,9 @@
"status",
"updatedAt"
],
- "type": "object"
+ "type": "object",
+ "x-go-name": "GettableSilence",
+ "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"gettableSilences": {
"items": {
@@ -2956,6 +2958,7 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"postableSilence": {
+ "description": "PostableSilence postable silence",
"properties": {
"comment": {
"description": "comment",
@@ -2995,12 +2998,9 @@
"matchers",
"startsAt"
],
- "type": "object",
- "x-go-name": "PostableSilence",
- "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+ "type": "object"
},
"receiver": {
- "description": "Receiver receiver",
"properties": {
"name": {
"description": "name",
@@ -3011,7 +3011,9 @@
"required": [
"name"
],
- "type": "object"
+ "type": "object",
+ "x-go-name": "Receiver",
+ "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"silence": {
"description": "Silence silence",
@@ -3599,13 +3601,22 @@
"operationId": "RoutePostTestReceivers",
"parameters": [
{
- "in": "query",
- "items": {
- "$ref": "#/definitions/PostableApiReceiver"
- },
+ "in": "body",
"name": "receivers",
- "type": "array",
+ "schema": {
+ "items": {
+ "$ref": "#/definitions/PostableApiReceiver"
+ },
+ "type": "array"
+ },
"x-go-name": "Receivers"
+ },
+ {
+ "description": "Recipient should be \"grafana\" for requests to be handled by grafana\nand the numeric datasource id for requests to be forwarded to a datasource",
+ "in": "path",
+ "name": "Recipient",
+ "required": true,
+ "type": "string"
}
],
"responses": {
@@ -3616,7 +3627,10 @@
}
},
"207": {
- "$ref": "#/responses/MultiStatus"
+ "description": "MultiStatus",
+ "schema": {
+ "$ref": "#/definitions/MultiStatus"
+ }
},
"400": {
"description": "ValidationError",
@@ -3624,11 +3638,29 @@
"$ref": "#/definitions/ValidationError"
}
},
+ "403": {
+ "description": "PermissionDenied",
+ "schema": {
+ "$ref": "#/definitions/PermissionDenied"
+ }
+ },
+ "404": {
+ "description": "AlertManagerNotFound",
+ "schema": {
+ "$ref": "#/definitions/AlertManagerNotFound"
+ }
+ },
"408": {
"description": "Failure",
"schema": {
"$ref": "#/definitions/Failure"
}
+ },
+ "409": {
+ "description": "AlertManagerNotReady",
+ "schema": {
+ "$ref": "#/definitions/AlertManagerNotReady"
+ }
}
},
"summary": "Test Grafana managed receivers without saving them.",
@@ -4042,57 +4074,6 @@
]
}
},
- "/api/v1/receiver/test/{Recipient}": {
- "post": {
- "consumes": [
- "application/json"
- ],
- "description": "Test receiver",
- "operationId": "RouteTestReceiverConfig",
- "parameters": [
- {
- "description": "Recipient should be \"grafana\" for requests to be handled by grafana\nand the numeric datasource id for requests to be forwarded to a datasource",
- "in": "path",
- "name": "Recipient",
- "required": true,
- "type": "string"
- },
- {
- "in": "body",
- "name": "Body",
- "schema": {
- "$ref": "#/definitions/ExtendedReceiver"
- }
- }
- ],
- "produces": [
- "application/json"
- ],
- "responses": {
- "200": {
- "description": "Success",
- "schema": {
- "$ref": "#/definitions/Success"
- }
- },
- "412": {
- "description": "SmtpNotEnabled",
- "schema": {
- "$ref": "#/definitions/SmtpNotEnabled"
- }
- },
- "500": {
- "description": "Failure",
- "schema": {
- "$ref": "#/definitions/Failure"
- }
- }
- },
- "tags": [
- "testing"
- ]
- }
- },
"/api/v1/rule/test/{Recipient}": {
"post": {
"consumes": [
diff --git a/pkg/services/ngalert/api/tooling/spec.json b/pkg/services/ngalert/api/tooling/spec.json
index 648f972a350..6e89b3344de 100644
--- a/pkg/services/ngalert/api/tooling/spec.json
+++ b/pkg/services/ngalert/api/tooling/spec.json
@@ -494,13 +494,22 @@
"operationId": "RoutePostTestReceivers",
"parameters": [
{
- "type": "array",
- "items": {
- "$ref": "#/definitions/PostableApiReceiver"
- },
"x-go-name": "Receivers",
"name": "receivers",
- "in": "query"
+ "in": "body",
+ "schema": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/PostableApiReceiver"
+ }
+ }
+ },
+ {
+ "type": "string",
+ "description": "Recipient should be \"grafana\" for requests to be handled by grafana\nand the numeric datasource id for requests to be forwarded to a datasource",
+ "name": "Recipient",
+ "in": "path",
+ "required": true
}
],
"responses": {
@@ -511,7 +520,10 @@
}
},
"207": {
- "$ref": "#/responses/MultiStatus"
+ "description": "MultiStatus",
+ "schema": {
+ "$ref": "#/definitions/MultiStatus"
+ }
},
"400": {
"description": "ValidationError",
@@ -519,11 +531,29 @@
"$ref": "#/definitions/ValidationError"
}
},
+ "403": {
+ "description": "PermissionDenied",
+ "schema": {
+ "$ref": "#/definitions/PermissionDenied"
+ }
+ },
+ "404": {
+ "description": "AlertManagerNotFound",
+ "schema": {
+ "$ref": "#/definitions/AlertManagerNotFound"
+ }
+ },
"408": {
"description": "Failure",
"schema": {
"$ref": "#/definitions/Failure"
}
+ },
+ "409": {
+ "description": "AlertManagerNotReady",
+ "schema": {
+ "$ref": "#/definitions/AlertManagerNotReady"
+ }
}
}
}
@@ -933,57 +963,6 @@
}
}
},
- "/api/v1/receiver/test/{Recipient}": {
- "post": {
- "description": "Test receiver",
- "consumes": [
- "application/json"
- ],
- "produces": [
- "application/json"
- ],
- "tags": [
- "testing"
- ],
- "operationId": "RouteTestReceiverConfig",
- "parameters": [
- {
- "type": "string",
- "description": "Recipient should be \"grafana\" for requests to be handled by grafana\nand the numeric datasource id for requests to be forwarded to a datasource",
- "name": "Recipient",
- "in": "path",
- "required": true
- },
- {
- "name": "Body",
- "in": "body",
- "schema": {
- "$ref": "#/definitions/ExtendedReceiver"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Success",
- "schema": {
- "$ref": "#/definitions/Success"
- }
- },
- "412": {
- "description": "SmtpNotEnabled",
- "schema": {
- "$ref": "#/definitions/SmtpNotEnabled"
- }
- },
- "500": {
- "description": "Failure",
- "schema": {
- "$ref": "#/definitions/Failure"
- }
- }
- }
- }
- },
"/api/v1/rule/test/{Recipient}": {
"post": {
"description": "Test rule",
@@ -1107,6 +1086,14 @@
},
"x-go-package": "github.com/prometheus/client_golang/api/prometheus/v1"
},
+ "AlertManagerNotFound": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
+ "AlertManagerNotReady": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
"AlertManagersResult": {
"type": "object",
"title": "AlertManagersResult contains the result from querying the alertmanagers endpoint.",
@@ -2120,6 +2107,10 @@
},
"$ref": "#/definitions/Matchers"
},
+ "MultiStatus": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
"NamespaceConfigResponse": {
"type": "object",
"additionalProperties": {
@@ -2365,6 +2356,10 @@
},
"x-go-package": "github.com/prometheus/alertmanager/config"
},
+ "PermissionDenied": {
+ "type": "object",
+ "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+ },
"Point": {
"type": "object",
"title": "Point represents a single data point for a given timestamp.",
@@ -3311,19 +3306,6 @@
},
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
},
- "TestReceiversConfig": {
- "type": "object",
- "properties": {
- "receivers": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/PostableApiReceiver"
- },
- "x-go-name": "Receivers"
- }
- },
- "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
- },
"TestReceiversResult": {
"type": "object",
"properties": {
@@ -3369,9 +3351,8 @@
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
},
"URL": {
- "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
"type": "object",
- "title": "A URL represents a parsed URL (technically, a URI reference).",
+ "title": "URL is a custom URL type that allows validation at configuration load time.",
"properties": {
"ForceQuery": {
"type": "boolean"
@@ -3404,7 +3385,7 @@
"$ref": "#/definitions/Userinfo"
}
},
- "x-go-package": "net/url"
+ "x-go-package": "github.com/prometheus/common/config"
},
"Userinfo": {
"description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -3571,6 +3552,7 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"alertGroup": {
+ "description": "AlertGroup alert group",
"type": "object",
"required": [
"alerts",
@@ -3593,17 +3575,14 @@
"$ref": "#/definitions/receiver"
}
},
- "x-go-name": "AlertGroup",
- "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/alertGroup"
},
"alertGroups": {
+ "description": "AlertGroups alert groups",
"type": "array",
"items": {
"$ref": "#/definitions/alertGroup"
},
- "x-go-name": "AlertGroups",
- "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/alertGroups"
},
"alertStatus": {
@@ -3724,7 +3703,6 @@
"$ref": "#/definitions/Duration"
},
"gettableAlert": {
- "description": "GettableAlert gettable alert",
"type": "object",
"required": [
"labels",
@@ -3784,6 +3762,8 @@
"x-go-name": "UpdatedAt"
}
},
+ "x-go-name": "GettableAlert",
+ "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/gettableAlert"
},
"gettableAlerts": {
@@ -3796,7 +3776,6 @@
"$ref": "#/definitions/gettableAlerts"
},
"gettableSilence": {
- "description": "GettableSilence gettable silence",
"type": "object",
"required": [
"comment",
@@ -3849,6 +3828,8 @@
"x-go-name": "UpdatedAt"
}
},
+ "x-go-name": "GettableSilence",
+ "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/gettableSilence"
},
"gettableSilences": {
@@ -3987,6 +3968,7 @@
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
},
"postableSilence": {
+ "description": "PostableSilence postable silence",
"type": "object",
"required": [
"comment",
@@ -4027,12 +4009,9 @@
"x-go-name": "StartsAt"
}
},
- "x-go-name": "PostableSilence",
- "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/postableSilence"
},
"receiver": {
- "description": "Receiver receiver",
"type": "object",
"required": [
"name"
@@ -4044,6 +4023,8 @@
"x-go-name": "Name"
}
},
+ "x-go-name": "Receiver",
+ "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
"$ref": "#/definitions/receiver"
},
"silence": {
diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go
index 6f94a402bf8..ed6d3c53195 100644
--- a/pkg/services/ngalert/eval/eval.go
+++ b/pkg/services/ngalert/eval/eval.go
@@ -124,7 +124,8 @@ func GetExprRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time) (
OrgId: ctx.OrgID,
Headers: map[string]string{
// Some data sources check this in query method as sometimes alerting needs special considerations.
- "FromAlert": "true",
+ "FromAlert": "true",
+ "X-Cache-Skip": "true",
},
}
diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go
index dc68e16e76a..51c26bb4193 100644
--- a/pkg/services/ngalert/ngalert.go
+++ b/pkg/services/ngalert/ngalert.go
@@ -4,25 +4,25 @@ import (
"context"
"time"
- "github.com/benbjohnson/clock"
- "github.com/grafana/grafana/pkg/services/quota"
- "golang.org/x/sync/errgroup"
-
- "github.com/grafana/grafana/pkg/services/ngalert/api"
- "github.com/grafana/grafana/pkg/services/ngalert/eval"
- "github.com/grafana/grafana/pkg/services/ngalert/metrics"
- "github.com/grafana/grafana/pkg/services/ngalert/state"
- "github.com/grafana/grafana/pkg/services/ngalert/store"
-
"github.com/grafana/grafana/pkg/api/routing"
+ "github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/datasourceproxy"
"github.com/grafana/grafana/pkg/services/datasources"
+ "github.com/grafana/grafana/pkg/services/ngalert/api"
+ "github.com/grafana/grafana/pkg/services/ngalert/eval"
+ "github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
+ "github.com/grafana/grafana/pkg/services/ngalert/state"
+ "github.com/grafana/grafana/pkg/services/ngalert/store"
+ "github.com/grafana/grafana/pkg/services/quota"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
+
+ "github.com/benbjohnson/clock"
+ "golang.org/x/sync/errgroup"
)
const (
@@ -38,13 +38,14 @@ const (
)
func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, routeRegister routing.RouteRegister,
- sqlStore *sqlstore.SQLStore, dataService *tsdb.Service, dataProxy *datasourceproxy.DataSourceProxyService,
+ sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, dataService *tsdb.Service, dataProxy *datasourceproxy.DataSourceProxyService,
quotaService *quota.QuotaService, m *metrics.Metrics) (*AlertNG, error) {
ng := &AlertNG{
Cfg: cfg,
DataSourceCache: dataSourceCache,
RouteRegister: routeRegister,
SQLStore: sqlStore,
+ KVStore: kvStore,
DataService: dataService,
DataProxy: dataProxy,
QuotaService: quotaService,
@@ -69,6 +70,7 @@ type AlertNG struct {
DataSourceCache datasources.CacheService
RouteRegister routing.RouteRegister
SQLStore *sqlstore.SQLStore
+ KVStore kvstore.KVStore
DataService *tsdb.Service
DataProxy *datasourceproxy.DataSourceProxyService
QuotaService *quota.QuotaService
@@ -95,7 +97,7 @@ func (ng *AlertNG) init() error {
Logger: ng.Log,
}
- ng.MultiOrgAlertmanager = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store)
+ ng.MultiOrgAlertmanager = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore)
// Let's make sure we're able to complete an initial sync of Alertmanagers before we start the alerting components.
if err := ng.MultiOrgAlertmanager.LoadAndSyncAlertmanagersForOrgs(context.Background()); err != nil {
diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go
index da9f29b383b..891a2bb954e 100644
--- a/pkg/services/ngalert/notifier/alertmanager.go
+++ b/pkg/services/ngalert/notifier/alertmanager.go
@@ -27,6 +27,7 @@ import (
"github.com/prometheus/common/model"
"github.com/grafana/grafana/pkg/components/securejsondata"
+ "github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/logging"
@@ -38,6 +39,9 @@ import (
)
const (
+ notificationLogFilename = "notifications"
+ silencesFilename = "silences"
+
workingDir = "alerting"
// How long should we keep silences and notification entries on-disk after they've served their purpose.
retentionNotificationsAndSilences = 5 * 24 * time.Hour
@@ -77,9 +81,10 @@ type Alertmanager struct {
logger log.Logger
gokitLogger gokit_log.Logger
- Settings *setting.Cfg
- Store store.AlertingStore
- Metrics *metrics.Metrics
+ Settings *setting.Cfg
+ Store store.AlertingStore
+ fileStore *FileStore
+ Metrics *metrics.Metrics
notificationLog *nflog.Log
marker types.Marker
@@ -106,28 +111,39 @@ type Alertmanager struct {
orgID int64
}
-func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m *metrics.Metrics) (*Alertmanager, error) {
+func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, kvStore kvstore.KVStore, m *metrics.Metrics) (*Alertmanager, error) {
am := &Alertmanager{
Settings: cfg,
stopc: make(chan struct{}),
logger: log.New("alertmanager", "org", orgID),
marker: types.NewMarker(m.Registerer),
stageMetrics: notify.NewMetrics(m.Registerer),
- dispatcherMetrics: dispatch.NewDispatcherMetrics(m.Registerer),
+ dispatcherMetrics: dispatch.NewDispatcherMetrics(false, m.Registerer),
Store: store,
Metrics: m,
orgID: orgID,
}
am.gokitLogger = gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger))
+ am.fileStore = NewFileStore(am.orgID, kvStore, am.WorkingDirPath())
+
+ nflogFilepath, err := am.fileStore.FilepathFor(context.TODO(), notificationLogFilename)
+ if err != nil {
+ return nil, err
+ }
+ silencesFilePath, err := am.fileStore.FilepathFor(context.TODO(), silencesFilename)
+ if err != nil {
+ return nil, err
+ }
// Initialize the notification log
am.wg.Add(1)
- var err error
am.notificationLog, err = nflog.New(
nflog.WithRetention(retentionNotificationsAndSilences),
- nflog.WithSnapshot(filepath.Join(am.WorkingDirPath(), "notifications")),
- nflog.WithMaintenance(maintenanceNotificationAndSilences, am.stopc, am.wg.Done),
+ nflog.WithSnapshot(nflogFilepath),
+ nflog.WithMaintenance(maintenanceNotificationAndSilences, am.stopc, am.wg.Done, func() (int64, error) {
+ return am.fileStore.Persist(context.TODO(), notificationLogFilename, am.notificationLog)
+ }),
)
if err != nil {
return nil, fmt.Errorf("unable to initialize the notification log component of alerting: %w", err)
@@ -135,7 +151,7 @@ func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m
// Initialize silences
am.silences, err = silence.New(silence.Options{
Metrics: m.Registerer,
- SnapshotFile: filepath.Join(am.WorkingDirPath(), "silences"),
+ SnapshotFile: silencesFilePath,
Retention: retentionNotificationsAndSilences,
})
if err != nil {
@@ -144,12 +160,14 @@ func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m
am.wg.Add(1)
go func() {
- am.silences.Maintenance(15*time.Minute, filepath.Join(am.WorkingDirPath(), "silences"), am.stopc)
+ am.silences.Maintenance(15*time.Minute, silencesFilePath, am.stopc, func() (int64, error) {
+ return am.fileStore.Persist(context.TODO(), silencesFilename, am.silences)
+ })
am.wg.Done()
}()
// Initialize in-memory alerts
- am.alerts, err = mem.NewAlerts(context.Background(), am.marker, memoryAlertsGCInterval, am.gokitLogger)
+ am.alerts, err = mem.NewAlerts(context.Background(), am.marker, memoryAlertsGCInterval, nil, am.gokitLogger)
if err != nil {
return nil, fmt.Errorf("unable to initialize the alert provider component of alerting: %w", err)
}
@@ -390,7 +408,7 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
}
am.route = dispatch.NewRoute(cfg.AlertmanagerConfig.Route, nil)
- am.dispatcher = dispatch.NewDispatcher(am.alerts, am.route, routingStage, am.marker, timeoutFunc, am.gokitLogger, am.dispatcherMetrics)
+ am.dispatcher = dispatch.NewDispatcher(am.alerts, am.route, routingStage, am.marker, timeoutFunc, &nilLimits{}, am.gokitLogger, am.dispatcherMetrics)
am.wg.Add(1)
go func() {
@@ -707,3 +725,7 @@ func timeoutFunc(d time.Duration) time.Duration {
}
return d + waitFunc()
}
+
+type nilLimits struct{}
+
+func (n nilLimits) MaxNumberOfAggregationGroups() int { return 0 }
diff --git a/pkg/services/ngalert/notifier/alertmanager_test.go b/pkg/services/ngalert/notifier/alertmanager_test.go
index 744f6bf69c2..f3d1c504acf 100644
--- a/pkg/services/ngalert/notifier/alertmanager_test.go
+++ b/pkg/services/ngalert/notifier/alertmanager_test.go
@@ -47,7 +47,8 @@ func setupAMTest(t *testing.T) *Alertmanager {
Logger: log.New("alertmanager-test"),
}
- am, err := newAlertmanager(1, cfg, store, m)
+ kvStore := newFakeKVStore(t)
+ am, err := newAlertmanager(1, cfg, store, kvStore, m)
require.NoError(t, err)
return am
}
@@ -310,7 +311,7 @@ func TestPutAlert(t *testing.T) {
t.Run(c.title, func(t *testing.T) {
r := prometheus.NewRegistry()
am.marker = types.NewMarker(r)
- am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 15*time.Minute, gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger)))
+ am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 15*time.Minute, nil, gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger)))
require.NoError(t, err)
alerts := []*types.Alert{}
diff --git a/pkg/services/ngalert/notifier/channels/default_template.go b/pkg/services/ngalert/notifier/channels/default_template.go
index 1409e6d573a..cac318b896e 100644
--- a/pkg/services/ngalert/notifier/channels/default_template.go
+++ b/pkg/services/ngalert/notifier/channels/default_template.go
@@ -89,6 +89,9 @@ Labels:
func templateForTests(t *testing.T) *template.Template {
f, err := ioutil.TempFile("/tmp", "template")
require.NoError(t, err)
+ defer func(f *os.File) {
+ _ = f.Close()
+ }(f)
t.Cleanup(func() {
require.NoError(t, os.RemoveAll(f.Name()))
diff --git a/pkg/services/ngalert/notifier/channels/default_template_test.go b/pkg/services/ngalert/notifier/channels/default_template_test.go
index c5557a2c79a..1400cb280df 100644
--- a/pkg/services/ngalert/notifier/channels/default_template_test.go
+++ b/pkg/services/ngalert/notifier/channels/default_template_test.go
@@ -58,6 +58,9 @@ func TestDefaultTemplateString(t *testing.T) {
f, err := ioutil.TempFile("/tmp", "template")
require.NoError(t, err)
+ defer func(f *os.File) {
+ _ = f.Close()
+ }(f)
t.Cleanup(func() {
require.NoError(t, os.RemoveAll(f.Name()))
diff --git a/pkg/services/ngalert/notifier/file_store.go b/pkg/services/ngalert/notifier/file_store.go
new file mode 100644
index 00000000000..0e9ef84648c
--- /dev/null
+++ b/pkg/services/ngalert/notifier/file_store.go
@@ -0,0 +1,109 @@
+package notifier
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/grafana/grafana/pkg/infra/kvstore"
+)
+
+const KVNamespace = "alertmanager"
+
+// State represents any of the two 'states' of the alertmanager. Notification log or Silences.
+// MarshalBinary returns the binary representation of this internal state based on the protobuf.
+type State interface {
+ MarshalBinary() ([]byte, error)
+}
+
+// FileStore is in charge of persisting the alertmanager files to the database.
+// It uses the KVstore table and encodes the files as a base64 string.
+type FileStore struct {
+ kv *kvstore.NamespacedKVStore
+ orgID int64
+ workingDirPath string
+}
+
+func NewFileStore(orgID int64, store kvstore.KVStore, workingDirPath string) *FileStore {
+ return &FileStore{
+ workingDirPath: workingDirPath,
+ orgID: orgID,
+ kv: kvstore.WithNamespace(store, orgID, KVNamespace),
+ }
+}
+
+// FilepathFor returns the filepath to an Alertmanager file.
+// If the file is already present on disk it no-ops.
+// If not, it tries to read the database and if there's no file it no-ops.
+// If there is a file in the database, it decodes it and writes to disk for Alertmanager consumption.
+func (fs *FileStore) FilepathFor(ctx context.Context, filename string) (string, error) {
+ // If a file is already present, we'll use that one and eventually save it to the database.
+ // We don't need to do anything else.
+ if fs.IsExists(filename) {
+ return fs.pathFor(filename), nil
+ }
+
+ // Then, let's attempt to read it from the database.
+ content, exists, err := fs.kv.Get(ctx, filename)
+ if err != nil {
+ return "", fmt.Errorf("error reading file '%s' from database: %w", filename, err)
+ }
+
+ // if it doesn't exist, let's no-op and let the Alertmanager create one. We'll eventually save it to the database.
+ if !exists {
+ return fs.pathFor(filename), nil
+ }
+
+ // If we have a file stored in the database, let's decode it and write it to disk to perform that initial load to memory.
+ bytes, err := decode(content)
+ if err != nil {
+ return "", fmt.Errorf("error decoding file '%s': %w", filename, err)
+ }
+
+ if err := fs.WriteFileToDisk(filename, bytes); err != nil {
+ return "", fmt.Errorf("error writing file %s: %w", filename, err)
+ }
+
+ return fs.pathFor(filename), err
+}
+
+// Persist takes care of persisting the binary representation of internal state to the database as a base64 encoded string.
+func (fs *FileStore) Persist(ctx context.Context, filename string, st State) (int64, error) {
+ var size int64
+
+ bytes, err := st.MarshalBinary()
+ if err != nil {
+ return size, err
+ }
+
+ if err = fs.kv.Set(ctx, filename, encode(bytes)); err != nil {
+ return size, err
+ }
+
+ return int64(len(bytes)), err
+}
+
+// IsExists verifies if the file exists or not.
+func (fs *FileStore) IsExists(fn string) bool {
+ _, err := os.Stat(fs.pathFor(fn))
+ return os.IsExist(err)
+}
+
+// WriteFileToDisk writes a file with the provided name and contents to the Alertmanager working directory with the default grafana permission.
+func (fs *FileStore) WriteFileToDisk(fn string, content []byte) error {
+ return os.WriteFile(fs.pathFor(fn), content, 0644)
+}
+
+func (fs *FileStore) pathFor(fn string) string {
+ return filepath.Join(fs.workingDirPath, fn)
+}
+
+func decode(s string) ([]byte, error) {
+ return base64.StdEncoding.DecodeString(s)
+}
+
+func encode(b []byte) string {
+ return base64.StdEncoding.EncodeToString(b)
+}
diff --git a/pkg/services/ngalert/notifier/file_store_test.go b/pkg/services/ngalert/notifier/file_store_test.go
new file mode 100644
index 00000000000..6c47868c097
--- /dev/null
+++ b/pkg/services/ngalert/notifier/file_store_test.go
@@ -0,0 +1,74 @@
+package notifier
+
+import (
+ "context"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestFileStore_FilepathFor(t *testing.T) {
+ store := newFakeKVStore(t)
+ workingDir := t.TempDir()
+ fs := NewFileStore(1, store, workingDir)
+ filekey := "silences"
+ filePath := filepath.Join(workingDir, filekey)
+
+ // With a file already on disk, it returns the existing file's filepath and no modification to the original file.
+ {
+ require.NoError(t, os.WriteFile(filePath, []byte("silence1,silence2"), 0644))
+ r, err := fs.FilepathFor(context.Background(), filekey)
+ require.NoError(t, err)
+ require.Equal(t, filePath, r)
+ f, err := ioutil.ReadFile(filepath.Clean(filePath))
+ require.NoError(t, err)
+ require.Equal(t, "silence1,silence2", string(f))
+ require.NoError(t, os.Remove(filePath))
+ }
+
+ // With a file already on the database, it writes the file to disk and returns the filepath.
+ {
+ require.NoError(t, store.Set(context.Background(), 1, KVNamespace, filekey, encode([]byte("silence1,silence3"))))
+ r, err := fs.FilepathFor(context.Background(), filekey)
+ require.NoError(t, err)
+ require.Equal(t, filePath, r)
+ f, err := ioutil.ReadFile(filepath.Clean(filePath))
+ require.NoError(t, err)
+ require.Equal(t, "silence1,silence3", string(f))
+ require.NoError(t, os.Remove(filePath))
+ require.NoError(t, store.Del(context.Background(), 1, KVNamespace, filekey))
+ }
+
+ // With no file on disk or database, it returns the original filepath.
+ {
+ r, err := fs.FilepathFor(context.Background(), filekey)
+ require.NoError(t, err)
+ require.Equal(t, filePath, r)
+ _, err = ioutil.ReadFile(filepath.Clean(filePath))
+ require.Error(t, err)
+ }
+}
+
+func TestFileStore_Persist(t *testing.T) {
+ store := newFakeKVStore(t)
+ state := &fakeState{data: "something to marshal"}
+ workingDir := t.TempDir()
+ fs := NewFileStore(1, store, workingDir)
+ filekey := "silences"
+
+ size, err := fs.Persist(context.Background(), filekey, state)
+ require.NoError(t, err)
+ require.Equal(t, int64(20), size)
+ store.mtx.Lock()
+ require.Len(t, store.store, 1)
+ store.mtx.Unlock()
+ v, ok, err := store.Get(context.Background(), 1, KVNamespace, filekey)
+ require.NoError(t, err)
+ require.True(t, ok)
+ b, err := decode(v)
+ require.NoError(t, err)
+ require.Equal(t, "something to marshal", string(b))
+}
diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go
index a4fcdee08be..da9d11e3a0e 100644
--- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go
+++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go
@@ -6,6 +6,7 @@ import (
"sync"
"time"
+ "github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -13,10 +14,7 @@ import (
)
var (
- SyncOrgsPollInterval = 1 * time.Minute
-)
-
-var (
+ SyncOrgsPollInterval = 1 * time.Minute
ErrNoAlertmanagerForOrg = fmt.Errorf("Alertmanager does not exist for this organization")
ErrAlertmanagerNotReady = fmt.Errorf("Alertmanager is not ready yet")
)
@@ -30,17 +28,19 @@ type MultiOrgAlertmanager struct {
configStore store.AlertingStore
orgStore store.OrgStore
+ kvStore kvstore.KVStore
orgRegistry *metrics.OrgRegistries
}
-func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore) *MultiOrgAlertmanager {
+func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore, kvStore kvstore.KVStore) *MultiOrgAlertmanager {
return &MultiOrgAlertmanager{
settings: cfg,
logger: log.New("multiorg.alertmanager"),
alertmanagers: map[int64]*Alertmanager{},
configStore: configStore,
orgStore: orgStore,
+ kvStore: kvStore,
orgRegistry: metrics.NewOrgRegistries(),
}
}
@@ -86,7 +86,7 @@ func (moa *MultiOrgAlertmanager) SyncAlertmanagersForOrgs(orgIDs []int64) {
existing, found := moa.alertmanagers[orgID]
if !found {
reg := moa.orgRegistry.GetOrCreateOrgRegistry(orgID)
- am, err := newAlertmanager(orgID, moa.settings, moa.configStore, metrics.NewMetrics(reg))
+ am, err := newAlertmanager(orgID, moa.settings, moa.configStore, moa.kvStore, metrics.NewMetrics(reg))
if err != nil {
moa.logger.Error("unable to create Alertmanager for org", "org", orgID, "err", err)
}
diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
index 4209e11ed1a..2874301da3d 100644
--- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
+++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go
@@ -12,6 +12,7 @@ import (
)
func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
+ t.Skipf("Skipping multiorg alertmanager tests for now")
configStore := &FakeConfigStore{
configs: map[int64]*models.AlertConfiguration{},
}
@@ -19,7 +20,8 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
orgs: []int64{1, 2, 3},
}
SyncOrgsPollInterval = 10 * time.Minute // Don't poll in unit tests.
- mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore)
+ kvStore := newFakeKVStore(t)
+ mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore, kvStore)
ctx := context.Background()
// Ensure that one Alertmanager is created per org.
@@ -42,6 +44,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
}
func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
+ t.Skipf("Skipping multiorg alertmanager tests for now")
configStore := &FakeConfigStore{
configs: map[int64]*models.AlertConfiguration{},
}
@@ -50,7 +53,8 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
}
SyncOrgsPollInterval = 10 * time.Minute // Don't poll in unit tests.
- mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore)
+ kvStore := newFakeKVStore(t)
+ mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore, kvStore)
ctx := context.Background()
// Ensure that one Alertmanagers is created per org.
diff --git a/pkg/services/ngalert/notifier/testing.go b/pkg/services/ngalert/notifier/testing.go
index d8b8e2fc321..cc93778341a 100644
--- a/pkg/services/ngalert/notifier/testing.go
+++ b/pkg/services/ngalert/notifier/testing.go
@@ -2,6 +2,8 @@ package notifier
import (
"context"
+ "sync"
+ "testing"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -54,3 +56,76 @@ type FakeOrgStore struct {
func (f *FakeOrgStore) GetOrgs(_ context.Context) ([]int64, error) {
return f.orgs, nil
}
+
+type FakeKVStore struct {
+ mtx sync.Mutex
+ store map[int64]map[string]map[string]string
+}
+
+func newFakeKVStore(t *testing.T) *FakeKVStore {
+ t.Helper()
+
+ return &FakeKVStore{
+ store: map[int64]map[string]map[string]string{},
+ }
+}
+
+func (fkv *FakeKVStore) Get(_ context.Context, orgId int64, namespace string, key string) (string, bool, error) {
+ fkv.mtx.Lock()
+ defer fkv.mtx.Unlock()
+ org, ok := fkv.store[orgId]
+ if !ok {
+ return "", false, nil
+ }
+ k, ok := org[namespace]
+ if !ok {
+ return "", false, nil
+ }
+
+ v, ok := k[key]
+ if !ok {
+ return "", false, nil
+ }
+
+ return v, true, nil
+}
+func (fkv *FakeKVStore) Set(_ context.Context, orgId int64, namespace string, key string, value string) error {
+ fkv.mtx.Lock()
+ defer fkv.mtx.Unlock()
+ org, ok := fkv.store[orgId]
+ if !ok {
+ fkv.store[orgId] = map[string]map[string]string{}
+ }
+ _, ok = org[namespace]
+ if !ok {
+ fkv.store[orgId][namespace] = map[string]string{}
+ }
+
+ fkv.store[orgId][namespace][key] = value
+
+ return nil
+}
+func (fkv *FakeKVStore) Del(_ context.Context, orgId int64, namespace string, key string) error {
+ fkv.mtx.Lock()
+ defer fkv.mtx.Unlock()
+ org, ok := fkv.store[orgId]
+ if !ok {
+ return nil
+ }
+ _, ok = org[namespace]
+ if !ok {
+ return nil
+ }
+
+ delete(fkv.store[orgId][namespace], key)
+
+ return nil
+}
+
+type fakeState struct {
+ data string
+}
+
+func (fs *fakeState) MarshalBinary() ([]byte, error) {
+ return []byte(fs.data), nil
+}
diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go
index 619ed97d5ef..02e2f6c72cd 100644
--- a/pkg/services/ngalert/schedule/schedule_unit_test.go
+++ b/pkg/services/ngalert/schedule/schedule_unit_test.go
@@ -238,7 +238,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
RuleStore: rs,
InstanceStore: is,
AdminConfigStore: acs,
- MultiOrgNotifier: notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}),
+ MultiOrgNotifier: notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}),
Logger: logger,
Metrics: metrics.NewMetrics(prometheus.NewRegistry()),
AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go
index 360ba0aa490..a1e9e437b66 100644
--- a/pkg/services/ngalert/state/cache.go
+++ b/pkg/services/ngalert/state/cache.go
@@ -3,6 +3,7 @@ package state
import (
"bytes"
"fmt"
+ "math"
"strconv"
"strings"
"sync"
@@ -110,16 +111,13 @@ func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, la
// and labels template.
type templateCaptureValue struct {
Labels map[string]string
- Value *float64
+ Value float64
}
// String implements the Stringer interface to print the value of each RefID
// in the template via {{ $values.A }} rather than {{ $values.A.Value }}.
func (v templateCaptureValue) String() string {
- if v.Value != nil {
- return strconv.FormatFloat(*v.Value, 'f', -1, 64)
- }
- return "null"
+ return strconv.FormatFloat(v.Value, 'f', -1, 64)
}
func expandTemplate(name, text string, labels map[string]string, alertInstance eval.Result) (result string, resultErr error) {
@@ -148,23 +146,31 @@ func expandTemplate(name, text string, labels map[string]string, alertInstance e
Value string
}{
Labels: labels,
- Values: func() map[string]templateCaptureValue {
- m := make(map[string]templateCaptureValue)
- for k, v := range alertInstance.Values {
- m[k] = templateCaptureValue{
- Labels: v.Labels,
- Value: v.Value,
- }
- }
- return m
- }(),
- Value: alertInstance.EvaluationString,
+ Values: newTemplateCaptureValues(alertInstance.Values),
+ Value: alertInstance.EvaluationString,
}); err != nil {
return "", fmt.Errorf("error executing template %v: %s", name, err.Error())
}
return buffer.String(), nil
}
+func newTemplateCaptureValues(values map[string]eval.NumberValueCapture) map[string]templateCaptureValue {
+ m := make(map[string]templateCaptureValue)
+ for k, v := range values {
+ var f float64
+ if v.Value != nil {
+ f = *v.Value
+ } else {
+ f = math.NaN()
+ }
+ m[k] = templateCaptureValue{
+ Labels: v.Labels,
+ Value: f,
+ }
+ }
+ return m
+}
+
func (c *cache) set(entry *State) {
c.mtxStates.Lock()
defer c.mtxStates.Unlock()
diff --git a/pkg/services/ngalert/state/cache_test.go b/pkg/services/ngalert/state/cache_test.go
index 946a2c6352b..9f553105ced 100644
--- a/pkg/services/ngalert/state/cache_test.go
+++ b/pkg/services/ngalert/state/cache_test.go
@@ -17,16 +17,16 @@ func TestTemplateCaptureValueStringer(t *testing.T) {
value templateCaptureValue
expected string
}{{
- name: "nil value returns null",
- value: templateCaptureValue{Value: nil},
- expected: "null",
+ name: "0 is returned as integer value",
+ value: templateCaptureValue{Value: 0},
+ expected: "0",
}, {
name: "1.0 is returned as integer value",
- value: templateCaptureValue{Value: ptr.Float64(1.0)},
+ value: templateCaptureValue{Value: 1.0},
expected: "1",
}, {
name: "1.1 is returned as decimal value",
- value: templateCaptureValue{Value: ptr.Float64(1.1)},
+ value: templateCaptureValue{Value: 1.1},
expected: "1.1",
}}
@@ -46,12 +46,12 @@ func TestExpandTemplate(t *testing.T) {
expected string
expectedError error
}{{
- name: "instance labels are expanded into $labels",
+ name: "labels are expanded into $labels",
text: "{{ $labels.instance }} is down",
labels: data.Labels{"instance": "foo"},
expected: "foo is down",
}, {
- name: "missing instance label returns error",
+ name: "missing label in $labels returns error",
text: "{{ $labels.instance }} is down",
labels: data.Labels{},
expectedError: errors.New("error executing template __alert_test: template: __alert_test:1:86: executing \"__alert_test\" at <$labels.instance>: map has no entry for key \"instance\""),
@@ -63,11 +63,24 @@ func TestExpandTemplate(t *testing.T) {
"A": {
Var: "A",
Labels: data.Labels{"instance": "foo"},
- Value: ptr.Float64(10),
+ Value: ptr.Float64(1),
},
},
},
- expected: "foo has value 10",
+ expected: "foo has value 1",
+ }, {
+ name: "values can be passed to template functions such as printf",
+ text: "{{ $values.A.Labels.instance }} has value {{ $values.A.Value | printf \"%.1f\" }}",
+ alertInstance: eval.Result{
+ Values: map[string]eval.NumberValueCapture{
+ "A": {
+ Var: "A",
+ Labels: data.Labels{"instance": "foo"},
+ Value: ptr.Float64(1.1),
+ },
+ },
+ },
+ expected: "foo has value 1.1",
}, {
name: "missing label in $values returns error",
text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
@@ -76,13 +89,26 @@ func TestExpandTemplate(t *testing.T) {
"A": {
Var: "A",
Labels: data.Labels{},
- Value: ptr.Float64(10),
+ Value: ptr.Float64(1),
},
},
},
expectedError: errors.New("error executing template __alert_test: template: __alert_test:1:86: executing \"__alert_test\" at <$values.A.Labels.instance>: map has no entry for key \"instance\""),
}, {
- name: "value string is expanded into $value",
+ name: "missing value in $values is returned as NaN",
+ text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
+ alertInstance: eval.Result{
+ Values: map[string]eval.NumberValueCapture{
+ "A": {
+ Var: "A",
+ Labels: data.Labels{"instance": "foo"},
+ Value: nil,
+ },
+ },
+ },
+ expected: "foo has value NaN",
+ }, {
+ name: "assert value string is expanded into $value",
text: "{{ $value }}",
alertInstance: eval.Result{
EvaluationString: "[ var='A' labels={instance=foo} value=10 ]",
diff --git a/pkg/services/ngalert/tests/util.go b/pkg/services/ngalert/tests/util.go
index 6198ea4399f..61a903a9400 100644
--- a/pkg/services/ngalert/tests/util.go
+++ b/pkg/services/ngalert/tests/util.go
@@ -35,8 +35,7 @@ func SetupTestEnv(t *testing.T, baseInterval time.Duration) (*ngalert.AlertNG, *
cfg.FeatureToggles = map[string]bool{"ngalert": true}
m := metrics.NewMetrics(prometheus.NewRegistry())
- ng, err := ngalert.ProvideService(cfg, nil, routing.NewRouteRegister(), sqlstore.InitTestDB(t), nil, nil, nil,
- m)
+ ng, err := ngalert.ProvideService(cfg, nil, routing.NewRouteRegister(), sqlstore.InitTestDB(t), nil, nil, nil, nil, m)
require.NoError(t, err)
return ng, &store.DBstore{
SQLStore: ng.SQLStore,
diff --git a/pkg/services/sqlstore/migrations/libraryelements.go b/pkg/services/sqlstore/migrations/libraryelements.go
index 099e204fc1e..a60d6a72403 100644
--- a/pkg/services/sqlstore/migrations/libraryelements.go
+++ b/pkg/services/sqlstore/migrations/libraryelements.go
@@ -50,4 +50,8 @@ func addLibraryElementsMigrations(mg *migrator.Migrator) {
mg.AddMigration("create "+models.LibraryElementConnectionTableName+" table v1", migrator.NewAddTableMigration(libraryElementConnectionV1))
mg.AddMigration("add index "+models.LibraryElementConnectionTableName+" element_id-kind-connection_id", migrator.NewAddIndexMigration(libraryElementConnectionV1, libraryElementConnectionV1.Indices[0]))
+
+ mg.AddMigration("add unique index library_element org_id_uid", migrator.NewAddIndexMigration(libraryElementsV1, &migrator.Index{
+ Cols: []string{"org_id", "uid"}, Type: migrator.UniqueIndex,
+ }))
}
diff --git a/pkg/services/sqlstore/migrations/ualert/channel.go b/pkg/services/sqlstore/migrations/ualert/channel.go
index cae250776e6..e75b3ec8df5 100644
--- a/pkg/services/sqlstore/migrations/ualert/channel.go
+++ b/pkg/services/sqlstore/migrations/ualert/channel.go
@@ -125,14 +125,17 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, orgID int64, channelUid
m.migratedChannelsPerOrg[orgID] = make(map[*notificationChannel]struct{})
}
m.migratedChannelsPerOrg[orgID][c] = struct{}{}
- settings, secureSettings := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings)
+ settings, decryptedSecureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings)
+ if err != nil {
+ return err
+ }
portedChannels = append(portedChannels, &PostableGrafanaReceiver{
UID: uid,
Name: c.Name,
Type: c.Type,
DisableResolveMessage: c.DisableResolveMessage,
Settings: settings,
- SecureSettings: secureSettings,
+ SecureSettings: decryptedSecureSettings,
})
return nil
@@ -293,14 +296,17 @@ func (m *migration) addUnmigratedChannels(orgID int64, amConfigs *PostableUserCo
}
m.migratedChannelsPerOrg[orgID][c] = struct{}{}
- settings, secureSettings := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings)
+ settings, decryptedSecureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings)
+ if err != nil {
+ return err
+ }
portedChannels = append(portedChannels, &PostableGrafanaReceiver{
UID: uid,
Name: c.Name,
Type: c.Type,
DisableResolveMessage: c.DisableResolveMessage,
Settings: settings,
- SecureSettings: secureSettings,
+ SecureSettings: decryptedSecureSettings,
})
}
receiver.GrafanaManagedReceivers = portedChannels
@@ -326,7 +332,7 @@ func (m *migration) generateChannelUID() (string, bool) {
// Some settings were migrated from settings to secure settings in between.
// See https://grafana.com/docs/grafana/latest/installation/upgrading/#ensure-encryption-of-existing-alert-notification-channel-secrets.
// migrateSettingsToSecureSettings takes care of that.
-func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings securejsondata.SecureJsonData) (*simplejson.Json, map[string]string) {
+func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings securejsondata.SecureJsonData) (*simplejson.Json, map[string]string, error) {
keys := []string{}
switch chanType {
case "slack":
@@ -349,20 +355,28 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json,
keys = []string{"api_secret"}
}
- ss := secureSettings.Decrypt()
+ decryptedSecureSettings := secureSettings.Decrypt()
+ cloneSettings := simplejson.New()
+ settingsMap, err := settings.Map()
+ if err != nil {
+ return nil, nil, err
+ }
+ for k, v := range settingsMap {
+ cloneSettings.Set(k, v)
+ }
for _, k := range keys {
- if v, ok := ss[k]; ok && v != "" {
+ if v, ok := decryptedSecureSettings[k]; ok && v != "" {
continue
}
- sv := settings.Get(k).MustString()
+ sv := cloneSettings.Get(k).MustString()
if sv != "" {
- ss[k] = sv
- settings.Del(k)
+ decryptedSecureSettings[k] = sv
+ cloneSettings.Del(k)
}
}
- return settings, ss
+ return cloneSettings, decryptedSecureSettings, nil
}
func getLabelForRouteMatching(ruleUID string) (string, string) {
diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go
index 9ee97e866ac..0dd26b26942 100644
--- a/pkg/services/sqlstore/migrations/ualert/ualert.go
+++ b/pkg/services/sqlstore/migrations/ualert/ualert.go
@@ -405,6 +405,11 @@ func (m *rmMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
return err
}
+ _, err = sess.Exec("delete from ngalert_configuration")
+ if err != nil {
+ return err
+ }
+
_, err = sess.Exec("delete from alert_instance")
if err != nil {
return err
diff --git a/pkg/services/sqlstore/sql_test_data.go b/pkg/services/sqlstore/sql_test_data.go
deleted file mode 100644
index 83164d9c4e2..00000000000
--- a/pkg/services/sqlstore/sql_test_data.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package sqlstore
-
-import (
- "math/rand"
- "time"
-
- "github.com/grafana/grafana/pkg/bus"
- "github.com/grafana/grafana/pkg/models"
-)
-
-func init() {
- bus.AddHandler("sql", InsertSQLTestData)
-}
-
-func sqlRandomWalk(m1 string, m2 string, intWalker int64, floatWalker float64, sess *DBSession) error {
- timeWalker := time.Now().UTC().Add(time.Hour * -200)
- now := time.Now().UTC()
- step := time.Minute
-
- row := &models.SQLTestData{
- Metric1: m1,
- Metric2: m2,
- TimeEpoch: timeWalker.Unix(),
- TimeDateTime: timeWalker,
- }
-
- for timeWalker.Unix() < now.Unix() {
- timeWalker = timeWalker.Add(step)
-
- row.Id = 0
- row.ValueBigInt += rand.Int63n(200) - 100
- row.ValueDouble += rand.Float64() - 0.5
- row.ValueFloat += rand.Float32() - 0.5
- row.TimeEpoch = timeWalker.Unix()
- row.TimeDateTime = timeWalker
-
- sqlog.Info("Writing SQL test data row")
- if _, err := sess.Table("test_data").Insert(row); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func InsertSQLTestData(cmd *models.InsertSQLTestDataCommand) error {
- return inTransaction(func(sess *DBSession) error {
- var err error
-
- sqlog.Info("SQL TestData: Clearing previous test data")
- res, err := sess.Exec("TRUNCATE test_data")
- if err != nil {
- return err
- }
-
- rows, _ := res.RowsAffected()
- sqlog.Info("SQL TestData: Truncate done", "rows", rows)
-
- if err := sqlRandomWalk("server1", "frontend", 100, 1.123, sess); err != nil {
- return err
- }
- if err := sqlRandomWalk("server2", "frontend", 100, 1.123, sess); err != nil {
- return err
- }
- return sqlRandomWalk("server3", "frontend", 100, 1.123, sess)
- })
-}
diff --git a/pkg/services/sqlstore/stats.go b/pkg/services/sqlstore/stats.go
index 5b9dc4dc500..00d121d3530 100644
--- a/pkg/services/sqlstore/stats.go
+++ b/pkg/services/sqlstore/stats.go
@@ -19,6 +19,7 @@ func init() {
}
const activeUserTimeLimit = time.Hour * 24 * 30
+const dailyActiveUserTimeLimit = time.Hour * 24
func GetAlertNotifiersUsageStats(ctx context.Context, query *models.GetAlertNotifierUsageStatsQuery) error {
var rawSQL = `SELECT COUNT(*) AS count, type FROM ` + dialect.Quote("alert_notification") + ` GROUP BY type`
@@ -54,6 +55,9 @@ func GetSystemStats(query *models.GetSystemStatsQuery) error {
activeUserDeadlineDate := time.Now().Add(-activeUserTimeLimit)
sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE last_seen_at > ?) AS active_users,`, activeUserDeadlineDate)
+ dailyActiveUserDeadlineDate := time.Now().Add(-dailyActiveUserTimeLimit)
+ sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE last_seen_at > ?) AS daily_active_users,`, dailyActiveUserDeadlineDate)
+
sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboards,`, dialect.BooleanStr(false))
sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS folders,`, dialect.BooleanStr(true))
@@ -112,7 +116,10 @@ func roleCounterSQL() string {
strconv.FormatInt(userStatsCache.total.Viewers, 10) + ` AS viewers, ` +
strconv.FormatInt(userStatsCache.active.Admins, 10) + ` AS active_admins, ` +
strconv.FormatInt(userStatsCache.active.Editors, 10) + ` AS active_editors, ` +
- strconv.FormatInt(userStatsCache.active.Viewers, 10) + ` AS active_viewers`
+ strconv.FormatInt(userStatsCache.active.Viewers, 10) + ` AS active_viewers, ` +
+ strconv.FormatInt(userStatsCache.dailyActive.Admins, 10) + ` AS daily_active_admins, ` +
+ strconv.FormatInt(userStatsCache.dailyActive.Editors, 10) + ` AS daily_active_editors, ` +
+ strconv.FormatInt(userStatsCache.dailyActive.Viewers, 10) + ` AS daily_active_viewers`
return sqlQuery
}
@@ -131,6 +138,7 @@ func viewersPermissionsCounterSQL(statName string, isFolder bool, permission mod
func GetAdminStats(query *models.GetAdminStatsQuery) error {
activeEndDate := time.Now().Add(-activeUserTimeLimit)
+ dailyActiveEndDate := time.Now().Add(-dailyActiveUserTimeLimit)
var rawSQL = `SELECT
(
@@ -173,14 +181,22 @@ func GetAdminStats(query *models.GetAdminStatsQuery) error {
SELECT COUNT(*)
FROM ` + dialect.Quote("user") + ` WHERE last_seen_at > ?
) AS active_users,
+ (
+ SELECT COUNT(*)
+ FROM ` + dialect.Quote("user") + ` WHERE last_seen_at > ?
+ ) AS daily_active_users,
` + roleCounterSQL() + `,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("user_auth_token") + ` WHERE rotated_at > ?
- ) AS active_sessions`
+ ) AS active_sessions,
+ (
+ SELECT COUNT(*)
+ FROM ` + dialect.Quote("user_auth_token") + ` WHERE rotated_at > ?
+ ) AS daily_active_sessions`
var stats models.AdminStats
- _, err := x.SQL(rawSQL, activeEndDate, activeEndDate.Unix()).Get(&stats)
+ _, err := x.SQL(rawSQL, activeEndDate, dailyActiveEndDate, activeEndDate.Unix(), dailyActiveEndDate.Unix()).Get(&stats)
if err != nil {
return err
}
@@ -216,8 +232,9 @@ func updateUserRoleCountsIfNecessary(ctx context.Context, forced bool) error {
}
type memoUserStats struct {
- active models.UserStats
- total models.UserStats
+ active models.UserStats
+ dailyActive models.UserStats
+ total models.UserStats
memoized time.Time
}
@@ -230,7 +247,7 @@ var (
func updateUserRoleCounts(ctx context.Context) error {
query := `
SELECT role AS bitrole, active, COUNT(role) AS count FROM
- (SELECT last_seen_at>? AS active, SUM(role) AS role
+ (SELECT last_seen_at>? AS active, last_seen_at>? AS daily_active, SUM(role) AS role
FROM (SELECT
u.id,
CASE org_user.role
@@ -242,18 +259,20 @@ SELECT role AS bitrole, active, COUNT(role) AS count FROM
FROM ` + dialect.Quote("user") + ` AS u INNER JOIN org_user ON org_user.user_id = u.id
GROUP BY u.id, u.last_seen_at, org_user.role) AS t2
GROUP BY id, last_seen_at) AS t1
-GROUP BY active, role;`
+GROUP BY active, daily_active, role;`
activeUserDeadline := time.Now().Add(-activeUserTimeLimit)
+ dailyActiveUserDeadline := time.Now().Add(-dailyActiveUserTimeLimit)
type rolebitmap struct {
- Active bool
- Bitrole int64
- Count int64
+ Active bool
+ DailyActive bool
+ Bitrole int64
+ Count int64
}
bitmap := []rolebitmap{}
- err := x.Context(ctx).SQL(query, activeUserDeadline).Find(&bitmap)
+ err := x.Context(ctx).SQL(query, activeUserDeadline, dailyActiveUserDeadline).Find(&bitmap)
if err != nil {
return err
}
@@ -271,6 +290,9 @@ GROUP BY active, role;`
if role.Active {
memo.active = addToStats(memo.active, roletype, role.Count)
}
+ if role.DailyActive {
+ memo.dailyActive = addToStats(memo.dailyActive, roletype, role.Count)
+ }
}
userStatsCache = memo
diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go
index 0f1355a4a61..1bbd5fce7eb 100644
--- a/pkg/setting/setting.go
+++ b/pkg/setting/setting.go
@@ -326,6 +326,8 @@ type Cfg struct {
DataProxyMaxIdleConns int
DataProxyKeepAlive int
DataProxyIdleConnTimeout int
+ ResponseLimit int64
+ DataProxyRowLimit int64
// DistributedCache
RemoteCacheOptions *RemoteCacheOptions
@@ -446,6 +448,10 @@ func (cfg Cfg) IsHTTPRequestHistogramDisabled() bool {
return cfg.FeatureToggles["disable_http_request_histogram"]
}
+func (cfg Cfg) IsNewNavigationEnabled() bool {
+ return cfg.FeatureToggles["newNavigation"]
+}
+
type CommandLineArgs struct {
Config string
HomePath string
@@ -801,24 +807,6 @@ func NewCfgFromArgs(args CommandLineArgs) (*Cfg, error) {
return cfg, nil
}
-var theCfg *Cfg
-
-// GetCfg gets the Cfg singleton.
-// XXX: This is only required for integration tests so that the configuration can be reset for each test,
-// as due to how the current DI framework functions, we can't create a new Cfg object every time (the services
-// constituting the DI graph, and referring to a Cfg instance, get created only once).
-func GetCfg() *Cfg {
- if theCfg != nil {
- return theCfg
- }
-
- theCfg, err := NewCfgFromArgs(CommandLineArgs{})
- if err != nil {
- panic(err)
- }
- return theCfg
-}
-
func (cfg *Cfg) validateStaticRootPath() error {
if skipStaticRootValidation {
return nil
@@ -1486,10 +1474,6 @@ func (cfg *Cfg) GetContentDeliveryURL(prefix string) string {
url := *cfg.CDNRootURL
preReleaseFolder := ""
- if strings.Contains(cfg.BuildVersion, "pre") || strings.Contains(cfg.BuildVersion, "alpha") {
- preReleaseFolder = "pre-releases"
- }
-
url.Path = path.Join(url.Path, prefix, preReleaseFolder, cfg.BuildVersion)
return url.String() + "/"
}
diff --git a/pkg/setting/setting_data_proxy.go b/pkg/setting/setting_data_proxy.go
index 80593bbda74..f879b4c9093 100644
--- a/pkg/setting/setting_data_proxy.go
+++ b/pkg/setting/setting_data_proxy.go
@@ -2,6 +2,8 @@ package setting
import "gopkg.in/ini.v1"
+const defaultDataProxyRowLimit = int64(1000000)
+
func readDataProxySettings(iniFile *ini.File, cfg *Cfg) error {
dataproxy := iniFile.Section("dataproxy")
cfg.SendUserHeader = dataproxy.Key("send_user_header").MustBool(false)
@@ -14,6 +16,12 @@ func readDataProxySettings(iniFile *ini.File, cfg *Cfg) error {
cfg.DataProxyMaxConnsPerHost = dataproxy.Key("max_conns_per_host").MustInt(0)
cfg.DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt()
cfg.DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90)
+ cfg.ResponseLimit = dataproxy.Key("response_limit").MustInt64(0)
+ cfg.DataProxyRowLimit = dataproxy.Key("row_limit").MustInt64(defaultDataProxyRowLimit)
+
+ if cfg.DataProxyRowLimit <= 0 {
+ cfg.DataProxyRowLimit = defaultDataProxyRowLimit
+ }
if val, err := dataproxy.Key("max_idle_connections_per_host").Int(); err == nil {
cfg.Logger.Warn("[Deprecated] the configuration setting 'max_idle_connections_per_host' is deprecated, please use 'max_idle_connections' instead")
diff --git a/pkg/setting/setting_test.go b/pkg/setting/setting_test.go
index 486dd6ef47d..2336c780648 100644
--- a/pkg/setting/setting_test.go
+++ b/pkg/setting/setting_test.go
@@ -413,8 +413,8 @@ func TestGetCDNPathWithPreReleaseVersionAndSubPath(t *testing.T) {
cfg.BuildVersion = "v7.5.0-11124pre"
cfg.CDNRootURL, err = url.Parse("http://cdn.grafana.com/sub")
require.NoError(t, err)
- require.Equal(t, "http://cdn.grafana.com/sub/grafana-oss/pre-releases/v7.5.0-11124pre/", cfg.GetContentDeliveryURL("grafana-oss"))
- require.Equal(t, "http://cdn.grafana.com/sub/grafana/pre-releases/v7.5.0-11124pre/", cfg.GetContentDeliveryURL("grafana"))
+ require.Equal(t, "http://cdn.grafana.com/sub/grafana-oss/v7.5.0-11124pre/", cfg.GetContentDeliveryURL("grafana-oss"))
+ require.Equal(t, "http://cdn.grafana.com/sub/grafana/v7.5.0-11124pre/", cfg.GetContentDeliveryURL("grafana"))
}
// Adding a case for this in case we switch to proper semver version strings
@@ -424,6 +424,6 @@ func TestGetCDNPathWithAlphaVersion(t *testing.T) {
cfg.BuildVersion = "v7.5.0-alpha.11124"
cfg.CDNRootURL, err = url.Parse("http://cdn.grafana.com")
require.NoError(t, err)
- require.Equal(t, "http://cdn.grafana.com/grafana-oss/pre-releases/v7.5.0-alpha.11124/", cfg.GetContentDeliveryURL("grafana-oss"))
- require.Equal(t, "http://cdn.grafana.com/grafana/pre-releases/v7.5.0-alpha.11124/", cfg.GetContentDeliveryURL("grafana"))
+ require.Equal(t, "http://cdn.grafana.com/grafana-oss/v7.5.0-alpha.11124/", cfg.GetContentDeliveryURL("grafana-oss"))
+ require.Equal(t, "http://cdn.grafana.com/grafana/v7.5.0-alpha.11124/", cfg.GetContentDeliveryURL("grafana"))
}
diff --git a/pkg/tests/api/metrics/api_metrics_test.go b/pkg/tests/api/metrics/api_metrics_test.go
index 7344b293a0d..afe2b5a3a0f 100644
--- a/pkg/tests/api/metrics/api_metrics_test.go
+++ b/pkg/tests/api/metrics/api_metrics_test.go
@@ -129,9 +129,6 @@ func TestQueryCloudWatchLogs(t *testing.T) {
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{}),
},
- Meta: &data.FrameMeta{
- PreferredVisualization: "logs",
- },
},
}
diff --git a/pkg/tsdb/azuremonitor/aztokenprovider/token_provider.go b/pkg/tsdb/azuremonitor/aztokenprovider/token_provider.go
index c92e7698014..0bbc1b02931 100644
--- a/pkg/tsdb/azuremonitor/aztokenprovider/token_provider.go
+++ b/pkg/tsdb/azuremonitor/aztokenprovider/token_provider.go
@@ -6,6 +6,7 @@ import (
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/azcredentials"
@@ -140,13 +141,7 @@ func (c *managedIdentityTokenRetriever) Init() error {
}
func (c *managedIdentityTokenRetriever) GetAccessToken(ctx context.Context, scopes []string) (*AccessToken, error) {
- // Workaround for a bug in Azure SDK which mutates the passed array of scopes
- // See details https://github.com/Azure/azure-sdk-for-go/issues/15308
- arr := make([]string, len(scopes))
- copy(arr, scopes)
- scopes = arr
-
- accessToken, err := c.credential.GetToken(ctx, azcore.TokenRequestOptions{Scopes: scopes})
+ accessToken, err := c.credential.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes})
if err != nil {
return nil, err
}
@@ -177,7 +172,7 @@ func (c *clientSecretTokenRetriever) Init() error {
}
func (c *clientSecretTokenRetriever) GetAccessToken(ctx context.Context, scopes []string) (*AccessToken, error) {
- accessToken, err := c.credential.GetToken(ctx, azcore.TokenRequestOptions{Scopes: scopes})
+ accessToken, err := c.credential.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes})
if err != nil {
return nil, err
}
diff --git a/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go
index 61bff7b4ebd..d10dff28df7 100644
--- a/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go
+++ b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go
@@ -37,7 +37,7 @@ type AzureResourceGraphQuery struct {
TimeRange backend.TimeRange
}
-const argAPIVersion = "2021-03-01"
+const argAPIVersion = "2021-06-01-preview"
const argQueryProviderName = "/providers/Microsoft.ResourceGraph/resources"
func (e *AzureResourceGraphDatasource) resourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) {
diff --git a/pkg/tsdb/cloudmonitoring/time_series_query.go b/pkg/tsdb/cloudmonitoring/time_series_query.go
index 89275e21fff..86b9b6bfc88 100644
--- a/pkg/tsdb/cloudmonitoring/time_series_query.go
+++ b/pkg/tsdb/cloudmonitoring/time_series_query.go
@@ -45,11 +45,7 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) run(ctx context.Context, t
return queryResult, cloudMonitoringResponse{}, "", nil
}
intervalCalculator := interval.NewCalculator(interval.CalculatorOptions{})
- interval, err := intervalCalculator.Calculate(*tsdbQuery.TimeRange, time.Duration(timeSeriesQuery.IntervalMS/1000)*time.Second, "min")
- if err != nil {
- queryResult.Error = err
- return queryResult, cloudMonitoringResponse{}, "", nil
- }
+ interval := intervalCalculator.Calculate(*tsdbQuery.TimeRange, time.Duration(timeSeriesQuery.IntervalMS/1000)*time.Second)
timeFormat := "2006/01/02-15:04:05"
timeSeriesQuery.Query += fmt.Sprintf(" | graph_period %s | within d'%s', d'%s'", interval.Text, from.UTC().Format(timeFormat), to.UTC().Format(timeFormat))
diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go
index 81dd9ccb72e..b7f42b081ef 100644
--- a/pkg/tsdb/cloudwatch/annotation_query.go
+++ b/pkg/tsdb/cloudwatch/annotation_query.go
@@ -21,7 +21,7 @@ func (e *cloudWatchExecutor) executeAnnotationQuery(ctx context.Context, model *
namespace := model.Get("namespace").MustString("")
metricName := model.Get("metricName").MustString("")
dimensions := model.Get("dimensions").MustMap()
- statistics := parseStatistics(model)
+ statistic := model.Get("statistic").MustString()
period := int64(model.Get("period").MustInt(0))
if period == 0 && !usePrefixMatch {
period = 300
@@ -45,9 +45,9 @@ func (e *cloudWatchExecutor) executeAnnotationQuery(ctx context.Context, model *
if err != nil {
return nil, errutil.Wrap("failed to call cloudwatch:DescribeAlarms", err)
}
- alarmNames = filterAlarms(resp, namespace, metricName, dimensions, statistics, period)
+ alarmNames = filterAlarms(resp, namespace, metricName, dimensions, statistic, period)
} else {
- if region == "" || namespace == "" || metricName == "" || len(statistics) == 0 {
+ if region == "" || namespace == "" || metricName == "" || statistic == "" {
return result, errors.New("invalid annotations query")
}
@@ -64,21 +64,19 @@ func (e *cloudWatchExecutor) executeAnnotationQuery(ctx context.Context, model *
}
}
}
- for _, s := range statistics {
- params := &cloudwatch.DescribeAlarmsForMetricInput{
- Namespace: aws.String(namespace),
- MetricName: aws.String(metricName),
- Dimensions: qd,
- Statistic: aws.String(s),
- Period: aws.Int64(period),
- }
- resp, err := cli.DescribeAlarmsForMetric(params)
- if err != nil {
- return nil, errutil.Wrap("failed to call cloudwatch:DescribeAlarmsForMetric", err)
- }
- for _, alarm := range resp.MetricAlarms {
- alarmNames = append(alarmNames, alarm.AlarmName)
- }
+ params := &cloudwatch.DescribeAlarmsForMetricInput{
+ Namespace: aws.String(namespace),
+ MetricName: aws.String(metricName),
+ Dimensions: qd,
+ Statistic: aws.String(statistic),
+ Period: aws.Int64(period),
+ }
+ resp, err := cli.DescribeAlarmsForMetric(params)
+ if err != nil {
+ return nil, errutil.Wrap("failed to call cloudwatch:DescribeAlarmsForMetric", err)
+ }
+ for _, alarm := range resp.MetricAlarms {
+ alarmNames = append(alarmNames, alarm.AlarmName)
}
}
@@ -133,7 +131,7 @@ func transformAnnotationToTable(annotations []map[string]string, query backend.D
}
func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, metricName string,
- dimensions map[string]interface{}, statistics []string, period int64) []*string {
+ dimensions map[string]interface{}, statistic string, period int64) []*string {
alarmNames := make([]*string, 0)
for _, alarm := range alarms.MetricAlarms {
@@ -144,33 +142,24 @@ func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, met
continue
}
- match := true
+ matchDimension := true
if len(dimensions) != 0 {
if len(alarm.Dimensions) != len(dimensions) {
- match = false
+ matchDimension = false
} else {
for _, d := range alarm.Dimensions {
if _, ok := dimensions[*d.Name]; !ok {
- match = false
+ matchDimension = false
}
}
}
}
- if !match {
+ if !matchDimension {
continue
}
- if len(statistics) != 0 {
- found := false
- for _, s := range statistics {
- if *alarm.Statistic == s {
- found = true
- break
- }
- }
- if !found {
- continue
- }
+ if *alarm.Statistic != statistic {
+ continue
}
if period != 0 && *alarm.Period != period {
diff --git a/pkg/tsdb/cloudwatch/cloudwatch_query.go b/pkg/tsdb/cloudwatch/cloudwatch_query.go
index df0b27e1c94..28163e729e8 100644
--- a/pkg/tsdb/cloudwatch/cloudwatch_query.go
+++ b/pkg/tsdb/cloudwatch/cloudwatch_query.go
@@ -1,24 +1,27 @@
package cloudwatch
import (
+ "encoding/json"
+ "fmt"
+ "net/url"
"strings"
+ "time"
)
type cloudWatchQuery struct {
- RefId string
- Region string
- Id string
- Namespace string
- MetricName string
- Stats string
- Expression string
- ReturnData bool
- Dimensions map[string][]string
- Period int
- Alias string
- MatchExact bool
- UsedExpression string
- RequestExceededMaxLimit bool
+ RefId string
+ Region string
+ Id string
+ Namespace string
+ MetricName string
+ Statistic string
+ Expression string
+ ReturnData bool
+ Dimensions map[string][]string
+ Period int
+ Alias string
+ MatchExact bool
+ UsedExpression string
}
func (q *cloudWatchQuery) isMathExpression() bool {
@@ -69,3 +72,51 @@ func (q *cloudWatchQuery) isMultiValuedDimensionExpression() bool {
return false
}
+
+func (q *cloudWatchQuery) buildDeepLink(startTime time.Time, endTime time.Time) (string, error) {
+ if q.isMathExpression() {
+ return "", nil
+ }
+
+ link := &cloudWatchLink{
+ Title: q.RefId,
+ View: "timeSeries",
+ Stacked: false,
+ Region: q.Region,
+ Start: startTime.UTC().Format(time.RFC3339),
+ End: endTime.UTC().Format(time.RFC3339),
+ }
+
+ if q.isSearchExpression() {
+ link.Metrics = []interface{}{&metricExpression{Expression: q.UsedExpression}}
+ } else {
+ metricStat := []interface{}{q.Namespace, q.MetricName}
+ for dimensionKey, dimensionValues := range q.Dimensions {
+ metricStat = append(metricStat, dimensionKey, dimensionValues[0])
+ }
+ metricStat = append(metricStat, &metricStatMeta{
+ Stat: q.Statistic,
+ Period: q.Period,
+ })
+ link.Metrics = []interface{}{metricStat}
+ }
+
+ linkProps, err := json.Marshal(link)
+ if err != nil {
+ return "", fmt.Errorf("could not marshal link: %w", err)
+ }
+
+ url, err := url.Parse(fmt.Sprintf(`https://%s.console.aws.amazon.com/cloudwatch/deeplink.js`, q.Region))
+ if err != nil {
+ return "", fmt.Errorf("unable to parse CloudWatch console deep link")
+ }
+
+ fragment := url.Query()
+ fragment.Set("graph", string(linkProps))
+
+ query := url.Query()
+ query.Set("region", q.Region)
+ url.RawQuery = query.Encode()
+
+ return fmt.Sprintf(`%s#metricsV2:%s`, url.String(), fragment.Encode()), nil
+}
diff --git a/pkg/tsdb/cloudwatch/cloudwatch_query_test.go b/pkg/tsdb/cloudwatch/cloudwatch_query_test.go
index 2c3379c2124..f75a60c23ba 100644
--- a/pkg/tsdb/cloudwatch/cloudwatch_query_test.go
+++ b/pkg/tsdb/cloudwatch/cloudwatch_query_test.go
@@ -12,7 +12,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "SEARCH(someexpression)",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
}
@@ -26,7 +26,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
MatchExact: true,
@@ -44,7 +44,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
Dimensions: map[string][]string{
@@ -61,7 +61,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
Dimensions: map[string][]string{
@@ -79,7 +79,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
Dimensions: map[string][]string{
@@ -97,7 +97,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
MatchExact: false,
@@ -123,7 +123,7 @@ func TestCloudWatchQuery(t *testing.T) {
RefId: "A",
Region: "us-east-1",
Expression: "",
- Stats: "Average",
+ Statistic: "Average",
Period: 300,
Id: "id1",
MatchExact: false,
diff --git a/pkg/tsdb/cloudwatch/log_actions.go b/pkg/tsdb/cloudwatch/log_actions.go
index 99bb4fd44be..b2b0ee536ba 100644
--- a/pkg/tsdb/cloudwatch/log_actions.go
+++ b/pkg/tsdb/cloudwatch/log_actions.go
@@ -35,36 +35,12 @@ func (e *cloudWatchExecutor) executeLogActions(ctx context.Context, req *backend
return err
}
- // When a query of the form "stats ... by ..." is made, we want to return
- // one series per group defined in the query, but due to the format
- // the query response is in, there does not seem to be a way to tell
- // by the response alone if/how the results should be grouped.
- // Because of this, if the frontend sees that a "stats ... by ..." query is being made
- // the "statsGroups" parameter is sent along with the query to the backend so that we
- // can correctly group the CloudWatch logs response.
- statsGroups := model.Get("statsGroups").MustStringArray()
- if len(statsGroups) > 0 && len(dataframe.Fields) > 0 {
- groupedFrames, err := groupResults(dataframe, statsGroups)
- if err != nil {
- return err
- }
-
- resultChan <- backend.Responses{
- query.RefID: backend.DataResponse{Frames: groupedFrames},
- }
- return nil
+ groupedFrames, err := groupResponseFrame(dataframe, model.Get("statsGroups").MustStringArray())
+ if err != nil {
+ return err
}
-
- if dataframe.Meta != nil {
- dataframe.Meta.PreferredVisualization = "logs"
- } else {
- dataframe.Meta = &data.FrameMeta{
- PreferredVisualization: "logs",
- }
- }
-
resultChan <- backend.Responses{
- query.RefID: backend.DataResponse{Frames: data.Frames{dataframe}},
+ query.RefID: backend.DataResponse{Frames: groupedFrames},
}
return nil
})
diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go
index afce302d9d1..f52a7338d58 100644
--- a/pkg/tsdb/cloudwatch/log_actions_test.go
+++ b/pkg/tsdb/cloudwatch/log_actions_test.go
@@ -81,9 +81,6 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
- Meta: &data.FrameMeta{
- PreferredVisualization: "logs",
- },
},
},
},
@@ -142,9 +139,6 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
- Meta: &data.FrameMeta{
- PreferredVisualization: "logs",
- },
},
},
},
@@ -220,9 +214,6 @@ func TestQuery_GetLogGroupFields(t *testing.T) {
aws.Int64(100), aws.Int64(30), aws.Int64(55),
}),
},
- Meta: &data.FrameMeta{
- PreferredVisualization: "logs",
- },
}
expFrame.RefID = refID
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
@@ -357,7 +348,6 @@ func TestQuery_StartQuery(t *testing.T) {
Custom: map[string]interface{}{
"Region": "default",
},
- PreferredVisualization: "logs",
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
refID: {
@@ -431,9 +421,6 @@ func TestQuery_StopQuery(t *testing.T) {
Fields: []*data.Field{
data.NewField("success", nil, []bool{true}),
},
- Meta: &data.FrameMeta{
- PreferredVisualization: "logs",
- },
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
diff --git a/pkg/tsdb/cloudwatch/metric_data_input_builder.go b/pkg/tsdb/cloudwatch/metric_data_input_builder.go
index 5b5727b6106..231d56155d7 100644
--- a/pkg/tsdb/cloudwatch/metric_data_input_builder.go
+++ b/pkg/tsdb/cloudwatch/metric_data_input_builder.go
@@ -8,7 +8,7 @@ import (
)
func (e *cloudWatchExecutor) buildMetricDataInput(startTime time.Time, endTime time.Time,
- queries map[string]*cloudWatchQuery) (*cloudwatch.GetMetricDataInput, error) {
+ queries []*cloudWatchQuery) (*cloudwatch.GetMetricDataInput, error) {
metricDataInput := &cloudwatch.GetMetricDataInput{
StartTime: aws.Time(startTime),
EndTime: aws.Time(endTime),
diff --git a/pkg/tsdb/cloudwatch/metric_data_query_builder.go b/pkg/tsdb/cloudwatch/metric_data_query_builder.go
index add6efa6237..ab5cbed0505 100644
--- a/pkg/tsdb/cloudwatch/metric_data_query_builder.go
+++ b/pkg/tsdb/cloudwatch/metric_data_query_builder.go
@@ -20,7 +20,7 @@ func (e *cloudWatchExecutor) buildMetricDataQuery(query *cloudWatchQuery) (*clou
mdq.Expression = aws.String(query.Expression)
} else {
if query.isSearchExpression() {
- mdq.Expression = aws.String(buildSearchExpression(query, query.Stats))
+ mdq.Expression = aws.String(buildSearchExpression(query, query.Statistic))
} else {
mdq.MetricStat = &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
@@ -37,7 +37,7 @@ func (e *cloudWatchExecutor) buildMetricDataQuery(query *cloudWatchQuery) (*clou
Value: aws.String(values[0]),
})
}
- mdq.MetricStat.Stat = aws.String(query.Stats)
+ mdq.MetricStat.Stat = aws.String(query.Statistic)
}
}
diff --git a/pkg/tsdb/cloudwatch/query_row_response.go b/pkg/tsdb/cloudwatch/query_row_response.go
new file mode 100644
index 00000000000..0978bf6deba
--- /dev/null
+++ b/pkg/tsdb/cloudwatch/query_row_response.go
@@ -0,0 +1,49 @@
+package cloudwatch
+
+import "github.com/aws/aws-sdk-go/service/cloudwatch"
+
+// queryRowResponse represents the GetMetricData response for a query row in the query editor.
+type queryRowResponse struct {
+ ID string
+ RequestExceededMaxLimit bool
+ PartialData bool
+ Labels []string
+ HasArithmeticError bool
+ ArithmeticErrorMessage string
+ Metrics map[string]*cloudwatch.MetricDataResult
+ StatusCode string
+}
+
+func newQueryRowResponse(id string) queryRowResponse {
+ return queryRowResponse{
+ ID: id,
+ RequestExceededMaxLimit: false,
+ PartialData: false,
+ HasArithmeticError: false,
+ ArithmeticErrorMessage: "",
+ Labels: []string{},
+ Metrics: map[string]*cloudwatch.MetricDataResult{},
+ }
+}
+
+func (q *queryRowResponse) addMetricDataResult(mdr *cloudwatch.MetricDataResult) {
+ label := *mdr.Label
+ q.Labels = append(q.Labels, label)
+ q.Metrics[label] = mdr
+ q.StatusCode = *mdr.StatusCode
+}
+
+func (q *queryRowResponse) appendTimeSeries(mdr *cloudwatch.MetricDataResult) {
+ if _, exists := q.Metrics[*mdr.Label]; !exists {
+ q.Metrics[*mdr.Label] = &cloudwatch.MetricDataResult{}
+ }
+ metric := q.Metrics[*mdr.Label]
+ metric.Timestamps = append(metric.Timestamps, mdr.Timestamps...)
+ metric.Values = append(metric.Values, mdr.Values...)
+ q.StatusCode = *mdr.StatusCode
+}
+
+func (q *queryRowResponse) addArithmeticError(message *string) {
+ q.HasArithmeticError = true
+ q.ArithmeticErrorMessage = *message
+}
diff --git a/pkg/tsdb/cloudwatch/query_transformer.go b/pkg/tsdb/cloudwatch/query_transformer.go
deleted file mode 100644
index aefa806ba4b..00000000000
--- a/pkg/tsdb/cloudwatch/query_transformer.go
+++ /dev/null
@@ -1,236 +0,0 @@
-package cloudwatch
-
-import (
- "encoding/json"
- "fmt"
- "net/url"
- "sort"
- "strings"
- "time"
-
- "github.com/grafana/grafana-plugin-sdk-go/backend"
- "github.com/grafana/grafana-plugin-sdk-go/data"
-)
-
-// returns a map of queries with query id as key. In the case a q request query
-// has more than one statistic defined, one cloudwatchQuery will be created for each statistic.
-// If the query doesn't have an Id defined by the user, we'll give it an with format `query[RefId]`. In the case
-// the incoming query had more than one stat, it will ge an id like `query[RefId]_[StatName]`, eg queryC_Average
-func (e *cloudWatchExecutor) transformRequestQueriesToCloudWatchQueries(requestQueries []*requestQuery) (
- map[string]*cloudWatchQuery, error) {
- plog.Debug("Transforming CloudWatch request queries")
- cloudwatchQueries := make(map[string]*cloudWatchQuery)
- for _, requestQuery := range requestQueries {
- for _, stat := range requestQuery.Statistics {
- id := requestQuery.Id
- if id == "" {
- id = fmt.Sprintf("query%s", requestQuery.RefId)
- }
- if len(requestQuery.Statistics) > 1 {
- id = fmt.Sprintf("%s_%v", id, strings.ReplaceAll(*stat, ".", "_"))
- }
-
- if _, ok := cloudwatchQueries[id]; ok {
- return nil, fmt.Errorf("error in query %q - query ID %q is not unique", requestQuery.RefId, id)
- }
-
- query := &cloudWatchQuery{
- Id: id,
- RefId: requestQuery.RefId,
- Region: requestQuery.Region,
- Namespace: requestQuery.Namespace,
- MetricName: requestQuery.MetricName,
- Dimensions: requestQuery.Dimensions,
- Stats: *stat,
- Period: requestQuery.Period,
- Alias: requestQuery.Alias,
- Expression: requestQuery.Expression,
- ReturnData: requestQuery.ReturnData,
- MatchExact: requestQuery.MatchExact,
- }
- cloudwatchQueries[id] = query
- }
- }
-
- return cloudwatchQueries, nil
-}
-
-func (e *cloudWatchExecutor) transformQueryResponsesToQueryResult(cloudwatchResponses []*cloudwatchResponse, requestQueries []*requestQuery, startTime time.Time, endTime time.Time) (map[string]*backend.DataResponse, error) {
- responsesByRefID := make(map[string][]*cloudwatchResponse)
- refIDs := sort.StringSlice{}
- for _, res := range cloudwatchResponses {
- refIDs = append(refIDs, res.RefId)
- responsesByRefID[res.RefId] = append(responsesByRefID[res.RefId], res)
- }
- // Ensure stable results
- refIDs.Sort()
-
- results := make(map[string]*backend.DataResponse)
- for _, refID := range refIDs {
- responses := responsesByRefID[refID]
- queryResult := backend.DataResponse{}
- frames := make(data.Frames, 0, len(responses))
-
- requestExceededMaxLimit := false
- partialData := false
- var executedQueries []executedQuery
-
- for _, response := range responses {
- frames = append(frames, response.DataFrames...)
- requestExceededMaxLimit = requestExceededMaxLimit || response.RequestExceededMaxLimit
- partialData = partialData || response.PartialData
-
- if requestExceededMaxLimit {
- frames[0].AppendNotices(data.Notice{
- Severity: data.NoticeSeverityWarning,
- Text: "cloudwatch GetMetricData error: Maximum number of allowed metrics exceeded. Your search may have been limited",
- })
- }
-
- if partialData {
- frames[0].AppendNotices(data.Notice{
- Severity: data.NoticeSeverityWarning,
- Text: "cloudwatch GetMetricData error: Too many datapoints requested - your search has been limited. Please try to reduce the time range",
- })
- }
-
- executedQueries = append(executedQueries, executedQuery{
- Expression: response.Expression,
- ID: response.Id,
- Period: response.Period,
- })
- }
-
- sort.Slice(frames, func(i, j int) bool {
- return frames[i].Name < frames[j].Name
- })
-
- eq, err := json.Marshal(executedQueries)
- if err != nil {
- return nil, fmt.Errorf("could not marshal executedString struct: %w", err)
- }
-
- link, err := buildDeepLink(refID, requestQueries, executedQueries, startTime, endTime)
- if err != nil {
- return nil, fmt.Errorf("could not build deep link: %w", err)
- }
-
- createDataLinks := func(link string) []data.DataLink {
- return []data.DataLink{{
- Title: "View in CloudWatch console",
- TargetBlank: true,
- URL: link,
- }}
- }
-
- for _, frame := range frames {
- if frame.Meta != nil {
- frame.Meta.ExecutedQueryString = string(eq)
- } else {
- frame.Meta = &data.FrameMeta{
- ExecutedQueryString: string(eq),
- }
- }
-
- if link == "" || len(frame.Fields) < 2 {
- continue
- }
-
- if frame.Fields[1].Config == nil {
- frame.Fields[1].Config = &data.FieldConfig{}
- }
-
- frame.Fields[1].Config.Links = createDataLinks(link)
- }
-
- queryResult.Frames = frames
- results[refID] = &queryResult
- }
-
- return results, nil
-}
-
-// buildDeepLink generates a deep link from Grafana to the CloudWatch console. The link params are based on
-// metric(s) for a given query row in the Query Editor.
-func buildDeepLink(refID string, requestQueries []*requestQuery, executedQueries []executedQuery, startTime time.Time,
- endTime time.Time) (string, error) {
- if isMathExpression(executedQueries) {
- return "", nil
- }
-
- requestQuery := &requestQuery{}
- for _, rq := range requestQueries {
- if rq.RefId == refID {
- requestQuery = rq
- break
- }
- }
-
- metricItems := []interface{}{}
- cloudWatchLinkProps := &cloudWatchLink{
- Title: refID,
- View: "timeSeries",
- Stacked: false,
- Region: requestQuery.Region,
- Start: startTime.UTC().Format(time.RFC3339),
- End: endTime.UTC().Format(time.RFC3339),
- }
-
- expressions := []interface{}{}
- for _, meta := range executedQueries {
- if strings.Contains(meta.Expression, "SEARCH(") {
- expressions = append(expressions, &metricExpression{Expression: meta.Expression})
- }
- }
-
- if len(expressions) != 0 {
- cloudWatchLinkProps.Metrics = expressions
- } else {
- for _, stat := range requestQuery.Statistics {
- metricStat := []interface{}{requestQuery.Namespace, requestQuery.MetricName}
- for dimensionKey, dimensionValues := range requestQuery.Dimensions {
- metricStat = append(metricStat, dimensionKey, dimensionValues[0])
- }
- metricStat = append(metricStat, &metricStatMeta{
- Stat: *stat,
- Period: requestQuery.Period,
- })
- metricItems = append(metricItems, metricStat)
- }
- cloudWatchLinkProps.Metrics = metricItems
- }
-
- linkProps, err := json.Marshal(cloudWatchLinkProps)
- if err != nil {
- return "", fmt.Errorf("could not marshal link: %w", err)
- }
-
- url, err := url.Parse(fmt.Sprintf(`https://%s.console.aws.amazon.com/cloudwatch/deeplink.js`, requestQuery.Region))
- if err != nil {
- return "", fmt.Errorf("unable to parse CloudWatch console deep link")
- }
-
- fragment := url.Query()
- fragment.Set("", string(linkProps))
-
- q := url.Query()
- q.Set("region", requestQuery.Region)
- url.RawQuery = q.Encode()
-
- link := fmt.Sprintf(`%s#metricsV2:graph%s`, url.String(), fragment.Encode())
-
- return link, nil
-}
-
-func isMathExpression(executedQueries []executedQuery) bool {
- isMathExpression := false
- for _, query := range executedQueries {
- if strings.Contains(query.Expression, "SEARCH(") {
- return false
- } else if query.Expression != "" {
- isMathExpression = true
- }
- }
-
- return isMathExpression
-}
diff --git a/pkg/tsdb/cloudwatch/query_transformer_test.go b/pkg/tsdb/cloudwatch/query_transformer_test.go
deleted file mode 100644
index daf0cc674b3..00000000000
--- a/pkg/tsdb/cloudwatch/query_transformer_test.go
+++ /dev/null
@@ -1,249 +0,0 @@
-package cloudwatch
-
-import (
- "net/url"
- "testing"
- "time"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/grafana/grafana/pkg/setting"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestQueryTransformer(t *testing.T) {
- executor := newExecutor(nil, nil, &setting.Cfg{}, fakeSessionCache{})
- t.Run("One cloudwatchQuery is generated when its request query has one stat", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average"}),
- Period: 600,
- Id: "",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.NoError(t, err)
- assert.Len(t, res, 1)
- })
-
- t.Run("Two cloudwatchQuery is generated when there's two stats", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average", "Sum"}),
- Period: 600,
- Id: "",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.NoError(t, err)
- assert.Len(t, res, 2)
- })
- t.Run("id is given by user that will be used in the cloudwatch query", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average"}),
- Period: 600,
- Id: "myid",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.Nil(t, err)
- assert.Equal(t, len(res), 1)
- assert.Contains(t, res, "myid")
- })
-
- t.Run("ID is not given by user", func(t *testing.T) {
- t.Run("ID will be generated based on ref ID if query only has one stat", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average"}),
- Period: 600,
- Id: "",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.NoError(t, err)
- assert.Len(t, res, 1)
- assert.Contains(t, res, "queryD")
- })
-
- t.Run("ID will be generated based on ref and stat name if query has two stats", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average", "Sum"}),
- Period: 600,
- Id: "",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.NoError(t, err)
- assert.Len(t, res, 2)
- assert.Contains(t, res, "queryD_Sum")
- assert.Contains(t, res, "queryD_Average")
- })
- })
-
- t.Run("dot should be removed when query has more than one stat and one of them is a percentile", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average", "p46.32"}),
- Period: 600,
- Id: "",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.NoError(t, err)
- assert.Len(t, res, 2)
- assert.Contains(t, res, "queryD_p46_32")
- })
-
- t.Run("should return an error if two queries have the same id", func(t *testing.T) {
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average", "p46.32"}),
- Period: 600,
- Id: "myId",
- },
- {
- RefId: "E",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average", "p46.32"}),
- Period: 600,
- Id: "myId",
- },
- }
-
- res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries)
- require.Nil(t, res)
- assert.Error(t, err)
- })
-
- requestQueries := []*requestQuery{
- {
- RefId: "D",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Sum"}),
- Period: 600,
- Id: "myId",
- },
- {
- RefId: "E",
- Region: "us-east-1",
- Namespace: "ec2",
- MetricName: "CPUUtilization",
- Statistics: aws.StringSlice([]string{"Average", "p46.32"}),
- Period: 600,
- Id: "myId",
- },
- }
-
- t.Run("A deep link that reference two metric stat metrics is created based on a request query with two stats", func(t *testing.T) {
- start, err := time.Parse(time.RFC3339, "2018-03-15T13:00:00Z")
- require.NoError(t, err)
- end, err := time.Parse(time.RFC3339, "2018-03-18T13:34:00Z")
- require.NoError(t, err)
-
- executedQueries := []executedQuery{{
- Expression: ``,
- ID: "D",
- Period: 600,
- }}
-
- link, err := buildDeepLink("E", requestQueries, executedQueries, start, end)
- require.NoError(t, err)
-
- parsedURL, err := url.Parse(link)
- require.NoError(t, err)
-
- decodedLink, err := url.PathUnescape(parsedURL.String())
- require.NoError(t, err)
- expected := `https://us-east-1.console.aws.amazon.com/cloudwatch/deeplink.js?region=us-east-1#metricsV2:graph={"view":"timeSeries","stacked":false,"title":"E","start":"2018-03-15T13:00:00Z","end":"2018-03-18T13:34:00Z","region":"us-east-1","metrics":[["ec2","CPUUtilization",{"stat":"Average","period":600}],["ec2","CPUUtilization",{"stat":"p46.32","period":600}]]}`
- assert.Equal(t, expected, decodedLink)
- })
-
- t.Run("A deep link that reference an expression based metric is created based on a request query with one stat", func(t *testing.T) {
- start, err := time.Parse(time.RFC3339, "2018-03-15T13:00:00Z")
- require.NoError(t, err)
- end, err := time.Parse(time.RFC3339, "2018-03-18T13:34:00Z")
- require.NoError(t, err)
-
- executedQueries := []executedQuery{{
- Expression: `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization"', 'Sum', 600))`,
- ID: "D",
- Period: 600,
- }}
-
- link, err := buildDeepLink("E", requestQueries, executedQueries, start, end)
- require.NoError(t, err)
-
- parsedURL, err := url.Parse(link)
- require.NoError(t, err)
-
- decodedLink, err := url.PathUnescape(parsedURL.String())
- require.NoError(t, err)
-
- expected := `https://us-east-1.console.aws.amazon.com/cloudwatch/deeplink.js?region=us-east-1#metricsV2:graph={"view":"timeSeries","stacked":false,"title":"E","start":"2018-03-15T13:00:00Z","end":"2018-03-18T13:34:00Z","region":"us-east-1","metrics":[{"expression":"REMOVE_EMPTY(SEARCH('Namespace=\"AWS/EC2\"+MetricName=\"CPUUtilization\"',+'Sum',+600))"}]}`
- assert.Equal(t, expected, decodedLink)
- })
-
- t.Run("A deep link is not built in case any of the executedQueries are math expressions", func(t *testing.T) {
- start, err := time.Parse(time.RFC3339, "2018-03-15T13:00:00Z")
- require.NoError(t, err)
- end, err := time.Parse(time.RFC3339, "2018-03-18T13:34:00Z")
- require.NoError(t, err)
-
- executedQueries := []executedQuery{{
- Expression: `a * 2`,
- ID: "D",
- Period: 600,
- }}
-
- link, err := buildDeepLink("E", requestQueries, executedQueries, start, end)
- require.NoError(t, err)
-
- parsedURL, err := url.Parse(link)
- require.NoError(t, err)
-
- decodedLink, err := url.PathUnescape(parsedURL.String())
- require.NoError(t, err)
- assert.Equal(t, "", decodedLink)
- })
-}
diff --git a/pkg/tsdb/cloudwatch/request_parser.go b/pkg/tsdb/cloudwatch/request_parser.go
index e842bb96ad2..411d69e172a 100644
--- a/pkg/tsdb/cloudwatch/request_parser.go
+++ b/pkg/tsdb/cloudwatch/request_parser.go
@@ -10,15 +10,19 @@ import (
"strings"
"time"
- "github.com/aws/aws-sdk-go/aws"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
)
-// Parses the json queries and returns a requestQuery. The requestQuery has a 1 to 1 mapping to a query editor row
-func (e *cloudWatchExecutor) parseQueries(queries []backend.DataQuery, startTime time.Time, endTime time.Time) (map[string][]*requestQuery, error) {
- requestQueries := make(map[string][]*requestQuery)
- for _, query := range queries {
+// parseQueries parses the json queries and returns a map of cloudWatchQueries by region. The cloudWatchQuery has a 1 to 1 mapping to a query editor row
+func (e *cloudWatchExecutor) parseQueries(queries []backend.DataQuery, startTime time.Time, endTime time.Time) (map[string][]*cloudWatchQuery, error) {
+ requestQueries := make(map[string][]*cloudWatchQuery)
+ migratedQueries, err := migrateLegacyQuery(queries, startTime, endTime)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, query := range migratedQueries {
model, err := simplejson.NewJson(query.JSON)
if err != nil {
return nil, &queryError{err: err, RefID: query.RefID}
@@ -36,7 +40,7 @@ func (e *cloudWatchExecutor) parseQueries(queries []backend.DataQuery, startTime
}
if _, exist := requestQueries[query.Region]; !exist {
- requestQueries[query.Region] = make([]*requestQuery, 0)
+ requestQueries[query.Region] = []*cloudWatchQuery{}
}
requestQueries[query.Region] = append(requestQueries[query.Region], query)
}
@@ -44,7 +48,41 @@ func (e *cloudWatchExecutor) parseQueries(queries []backend.DataQuery, startTime
return requestQueries, nil
}
-func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time, endTime time.Time) (*requestQuery, error) {
+// migrateLegacyQuery migrates queries that has a `statistics` field to use the `statistic` field instead.
+// This migration is also done in the frontend, so this should only ever be needed for alerting queries
+// In case the query used more than one stat, the first stat in the slice will be used in the statistic field
+// Read more here https://github.com/grafana/grafana/issues/30629
+func migrateLegacyQuery(queries []backend.DataQuery, startTime time.Time, endTime time.Time) ([]*backend.DataQuery, error) {
+ migratedQueries := []*backend.DataQuery{}
+ for _, q := range queries {
+ query := q
+ model, err := simplejson.NewJson(query.JSON)
+ if err != nil {
+ return nil, err
+ }
+
+ _, err = model.Get("statistic").String()
+ // If there's not a statistic property in the json, we know it's the legacy format and then it has to be migrated
+ if err != nil {
+ stats, err := model.Get("statistics").StringArray()
+ if err != nil {
+ return nil, fmt.Errorf("query must have either statistic or statistics field")
+ }
+ model.Del("statistics")
+ model.Set("statistic", stats[0])
+ query.JSON, err = model.MarshalJSON()
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ migratedQueries = append(migratedQueries, &query)
+ }
+
+ return migratedQueries, nil
+}
+
+func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time, endTime time.Time) (*cloudWatchQuery, error) {
plog.Debug("Parsing request query", "query", model)
reNumber := regexp.MustCompile(`^\d+$`)
region, err := model.Get("region").String()
@@ -63,7 +101,11 @@ func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time
if err != nil {
return nil, fmt.Errorf("failed to parse dimensions: %v", err)
}
- statistics := parseStatistics(model)
+
+ statistic, err := model.Get("statistic").String()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse statistic: %v", err)
+ }
p := model.Get("period").MustString("")
var period int
@@ -94,6 +136,12 @@ func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time
}
id := model.Get("id").MustString("")
+ if id == "" {
+ // Why not just use refId if id is not specified in the frontend? When specifying an id in the editor,
+ // and alphabetical must be used. The id must be unique, so if an id like for example a, b or c would be used,
+ // it would likely collide with some ref id. That's why the `query` prefix is used.
+ id = fmt.Sprintf("query%s", refId)
+ }
expression := model.Get("expression").MustString("")
alias := model.Get("alias").MustString()
returnData := !model.Get("hide").MustBool(false)
@@ -107,19 +155,20 @@ func parseRequestQuery(model *simplejson.Json, refId string, startTime time.Time
matchExact := model.Get("matchExact").MustBool(true)
- return &requestQuery{
- RefId: refId,
- Region: region,
- Namespace: namespace,
- MetricName: metricName,
- Dimensions: dimensions,
- Statistics: aws.StringSlice(statistics),
- Period: period,
- Alias: alias,
- Id: id,
- Expression: expression,
- ReturnData: returnData,
- MatchExact: matchExact,
+ return &cloudWatchQuery{
+ RefId: refId,
+ Region: region,
+ Id: id,
+ Namespace: namespace,
+ MetricName: metricName,
+ Statistic: statistic,
+ Expression: expression,
+ ReturnData: returnData,
+ Dimensions: dimensions,
+ Period: period,
+ Alias: alias,
+ MatchExact: matchExact,
+ UsedExpression: "",
}, nil
}
@@ -136,15 +185,6 @@ func getRetainedPeriods(timeSince time.Duration) []int {
}
}
-func parseStatistics(model *simplejson.Json) []string {
- var statistics []string
- for _, s := range model.Get("statistics").MustArray() {
- statistics = append(statistics, s.(string))
- }
-
- return statistics
-}
-
func parseDimensions(model *simplejson.Json) (map[string][]string, error) {
parsedDimensions := make(map[string][]string)
for k, v := range model.Get("dimensions").MustMap() {
diff --git a/pkg/tsdb/cloudwatch/request_parser_test.go b/pkg/tsdb/cloudwatch/request_parser_test.go
index 9f01fcdd5df..0a5b5f98723 100644
--- a/pkg/tsdb/cloudwatch/request_parser_test.go
+++ b/pkg/tsdb/cloudwatch/request_parser_test.go
@@ -4,14 +4,51 @@ import (
"testing"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/plugins"
+ "github.com/grafana/grafana/pkg/tsdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRequestParser(t *testing.T) {
- timeRange := plugins.NewDataTimeRange("now-1h", "now-2h")
+ t.Run("Query migration ", func(t *testing.T) {
+ t.Run("legacy statistics field is migrated", func(t *testing.T) {
+ startTime := time.Now()
+ endTime := startTime.Add(2 * time.Hour)
+ oldQuery := &backend.DataQuery{
+ MaxDataPoints: 0,
+ QueryType: "timeSeriesQuery",
+ Interval: 0,
+ }
+ oldQuery.RefID = "A"
+ oldQuery.JSON = []byte(`{
+ "region": "us-east-1",
+ "namespace": "ec2",
+ "metricName": "CPUUtilization",
+ "dimensions": {
+ "InstanceId": ["test"]
+ },
+ "statistics": ["Average", "Sum"],
+ "period": "600",
+ "hide": false
+ }`)
+ migratedQueries, err := migrateLegacyQuery([]backend.DataQuery{*oldQuery}, startTime, endTime)
+ require.NoError(t, err)
+ assert.Equal(t, 1, len(migratedQueries))
+
+ migratedQuery := migratedQueries[0]
+ assert.Equal(t, "A", migratedQuery.RefID)
+ model, err := simplejson.NewJson(migratedQuery.JSON)
+ require.NoError(t, err)
+ assert.Equal(t, "Average", model.Get("statistic").MustString())
+ res, err := model.Get("statistic").Array()
+ assert.Error(t, err)
+ assert.Nil(t, res)
+ })
+ })
+
+ timeRange := tsdb.NewTimeRange("now-1h", "now-2h")
from, err := timeRange.ParseFrom()
require.NoError(t, err)
to, err := timeRange.ParseTo()
@@ -29,9 +66,9 @@ func TestRequestParser(t *testing.T) {
"InstanceId": []interface{}{"test"},
"InstanceType": []interface{}{"test2", "test3"},
},
- "statistics": []interface{}{"Average"},
- "period": "600",
- "hide": false,
+ "statistic": "Average",
+ "period": "600",
+ "hide": false,
})
res, err := parseRequestQuery(query, "ref1", from, to)
@@ -40,7 +77,7 @@ func TestRequestParser(t *testing.T) {
assert.Equal(t, "ref1", res.RefId)
assert.Equal(t, "ec2", res.Namespace)
assert.Equal(t, "CPUUtilization", res.MetricName)
- assert.Empty(t, res.Id)
+ assert.Equal(t, "queryref1", res.Id)
assert.Empty(t, res.Expression)
assert.Equal(t, 600, res.Period)
assert.True(t, res.ReturnData)
@@ -48,8 +85,7 @@ func TestRequestParser(t *testing.T) {
assert.Len(t, res.Dimensions["InstanceId"], 1)
assert.Len(t, res.Dimensions["InstanceType"], 2)
assert.Equal(t, "test3", res.Dimensions["InstanceType"][1])
- assert.Len(t, res.Statistics, 1)
- assert.Equal(t, "Average", *res.Statistics[0])
+ assert.Equal(t, "Average", res.Statistic)
})
t.Run("Old dimensions structure (backwards compatibility)", func(t *testing.T) {
@@ -64,9 +100,9 @@ func TestRequestParser(t *testing.T) {
"InstanceId": "test",
"InstanceType": "test2",
},
- "statistics": []interface{}{"Average"},
- "period": "600",
- "hide": false,
+ "statistic": "Average",
+ "period": "600",
+ "hide": false,
})
res, err := parseRequestQuery(query, "ref1", from, to)
@@ -75,7 +111,7 @@ func TestRequestParser(t *testing.T) {
assert.Equal(t, "ref1", res.RefId)
assert.Equal(t, "ec2", res.Namespace)
assert.Equal(t, "CPUUtilization", res.MetricName)
- assert.Empty(t, res.Id)
+ assert.Equal(t, "queryref1", res.Id)
assert.Empty(t, res.Expression)
assert.Equal(t, 600, res.Period)
assert.True(t, res.ReturnData)
@@ -83,7 +119,7 @@ func TestRequestParser(t *testing.T) {
assert.Len(t, res.Dimensions["InstanceId"], 1)
assert.Len(t, res.Dimensions["InstanceType"], 1)
assert.Equal(t, "test2", res.Dimensions["InstanceType"][0])
- assert.Equal(t, "Average", *res.Statistics[0])
+ assert.Equal(t, "Average", res.Statistic)
})
t.Run("Period defined in the editor by the user is being used when time range is short", func(t *testing.T) {
@@ -98,11 +134,11 @@ func TestRequestParser(t *testing.T) {
"InstanceId": "test",
"InstanceType": "test2",
},
- "statistics": []interface{}{"Average"},
- "hide": false,
+ "statistic": "Average",
+ "hide": false,
})
query.Set("period", "900")
- timeRange := plugins.NewDataTimeRange("now-1h", "now-2h")
+ timeRange := tsdb.NewTimeRange("now-1h", "now-2h")
from, err := timeRange.ParseFrom()
require.NoError(t, err)
to, err := timeRange.ParseTo()
@@ -125,9 +161,9 @@ func TestRequestParser(t *testing.T) {
"InstanceId": "test",
"InstanceType": "test2",
},
- "statistics": []interface{}{"Average"},
- "hide": false,
- "period": "auto",
+ "statistic": "Average",
+ "hide": false,
+ "period": "auto",
})
t.Run("Time range is 5 minutes", func(t *testing.T) {
diff --git a/pkg/tsdb/cloudwatch/response_parser.go b/pkg/tsdb/cloudwatch/response_parser.go
index 6dec6513171..0e36d309ef0 100644
--- a/pkg/tsdb/cloudwatch/response_parser.go
+++ b/pkg/tsdb/cloudwatch/response_parser.go
@@ -8,86 +8,119 @@ import (
"time"
"github.com/aws/aws-sdk-go/service/cloudwatch"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/grafana/grafana/pkg/components/simplejson"
)
-func (e *cloudWatchExecutor) parseResponse(metricDataOutputs []*cloudwatch.GetMetricDataOutput,
- queries map[string]*cloudWatchQuery) ([]*cloudwatchResponse, error) {
- // Map from result ID -> label -> result
- mdrs := make(map[string]map[string]*cloudwatch.MetricDataResult)
- labels := map[string][]string{}
- for _, mdo := range metricDataOutputs {
- requestExceededMaxLimit := false
- for _, message := range mdo.Messages {
- if *message.Code == "MaxMetricsExceeded" {
- requestExceededMaxLimit = true
- }
- }
-
- for _, r := range mdo.MetricDataResults {
- id := *r.Id
- label := *r.Label
- if _, exists := mdrs[id]; !exists {
- mdrs[id] = make(map[string]*cloudwatch.MetricDataResult)
- mdrs[id][label] = r
- labels[id] = append(labels[id], label)
- } else if _, exists := mdrs[id][label]; !exists {
- mdrs[id][label] = r
- labels[id] = append(labels[id], label)
- } else {
- mdr := mdrs[id][label]
- mdr.Timestamps = append(mdr.Timestamps, r.Timestamps...)
- mdr.Values = append(mdr.Values, r.Values...)
- if *r.StatusCode == "Complete" {
- mdr.StatusCode = r.StatusCode
- }
- }
- queries[id].RequestExceededMaxLimit = requestExceededMaxLimit
- }
+func (e *cloudWatchExecutor) parseResponse(startTime time.Time, endTime time.Time, metricDataOutputs []*cloudwatch.GetMetricDataOutput,
+ queries []*cloudWatchQuery) ([]*responseWrapper, error) {
+ aggregatedResponse := aggregateResponse(metricDataOutputs)
+ queriesById := map[string]*cloudWatchQuery{}
+ for _, query := range queries {
+ queriesById[query.Id] = query
}
- cloudWatchResponses := make([]*cloudwatchResponse, 0, len(mdrs))
- for id, lr := range mdrs {
- query := queries[id]
- frames, partialData, err := parseMetricResults(lr, labels[id], query)
+ results := []*responseWrapper{}
+ for id, response := range aggregatedResponse {
+ queryRow := queriesById[id]
+ dataRes := backend.DataResponse{}
+
+ if response.HasArithmeticError {
+ dataRes.Error = fmt.Errorf("ArithmeticError in query %q: %s", queryRow.RefId, response.ArithmeticErrorMessage)
+ }
+
+ var err error
+ dataRes.Frames, err = buildDataFrames(startTime, endTime, response, queryRow)
if err != nil {
return nil, err
}
- response := &cloudwatchResponse{
- DataFrames: frames,
- Period: query.Period,
- Expression: query.UsedExpression,
- RefId: query.RefId,
- Id: query.Id,
- RequestExceededMaxLimit: query.RequestExceededMaxLimit,
- PartialData: partialData,
- }
- cloudWatchResponses = append(cloudWatchResponses, response)
+ results = append(results, &responseWrapper{
+ DataResponse: &dataRes,
+ RefId: queryRow.RefId,
+ })
}
- return cloudWatchResponses, nil
+ return results, nil
}
-func parseMetricResults(results map[string]*cloudwatch.MetricDataResult, labels []string,
- query *cloudWatchQuery) (data.Frames, bool, error) {
- partialData := false
- frames := data.Frames{}
- for _, label := range labels {
- result := results[label]
- if *result.StatusCode != "Complete" {
- partialData = true
- }
-
- for _, message := range result.Messages {
- if *message.Code == "ArithmeticError" {
- return nil, false, fmt.Errorf("ArithmeticError in query %q: %s", query.RefId, *message.Value)
+func aggregateResponse(getMetricDataOutputs []*cloudwatch.GetMetricDataOutput) map[string]queryRowResponse {
+ responseByID := make(map[string]queryRowResponse)
+ for _, gmdo := range getMetricDataOutputs {
+ requestExceededMaxLimit := false
+ for _, message := range gmdo.Messages {
+ if *message.Code == "MaxMetricsExceeded" {
+ requestExceededMaxLimit = true
}
}
+ for _, r := range gmdo.MetricDataResults {
+ id := *r.Id
+ label := *r.Label
+
+ response := newQueryRowResponse(id)
+ if _, exists := responseByID[id]; exists {
+ response = responseByID[id]
+ }
+
+ for _, message := range r.Messages {
+ if *message.Code == "ArithmeticError" {
+ response.addArithmeticError(message.Value)
+ }
+ }
+
+ if _, exists := response.Metrics[label]; !exists {
+ response.addMetricDataResult(r)
+ } else {
+ response.appendTimeSeries(r)
+ }
+
+ response.RequestExceededMaxLimit = response.RequestExceededMaxLimit || requestExceededMaxLimit
+ responseByID[id] = response
+ }
+ }
+
+ return responseByID
+}
+
+func getLabels(cloudwatchLabel string, query *cloudWatchQuery) data.Labels {
+ dims := make([]string, 0, len(query.Dimensions))
+ for k := range query.Dimensions {
+ dims = append(dims, k)
+ }
+ sort.Strings(dims)
+ labels := data.Labels{}
+ for _, dim := range dims {
+ values := query.Dimensions[dim]
+ if len(values) == 1 && values[0] != "*" {
+ labels[dim] = values[0]
+ } else {
+ for _, value := range values {
+ if value == cloudwatchLabel || value == "*" {
+ labels[dim] = cloudwatchLabel
+ } else if strings.Contains(cloudwatchLabel, value) {
+ labels[dim] = value
+ }
+ }
+ }
+ }
+ return labels
+}
+
+func buildDataFrames(startTime time.Time, endTime time.Time, aggregatedResponse queryRowResponse,
+ query *cloudWatchQuery) (data.Frames, error) {
+ frames := data.Frames{}
+ for _, label := range aggregatedResponse.Labels {
+ metric := aggregatedResponse.Metrics[label]
+
+ deepLink, err := query.buildDeepLink(startTime, endTime)
+ if err != nil {
+ return nil, err
+ }
// In case a multi-valued dimension is used and the cloudwatch query yields no values, create one empty time
// series for each dimension value. Use that dimension value to expand the alias field
- if len(result.Values) == 0 && query.isMultiValuedDimensionExpression() {
+ if len(metric.Values) == 0 && query.isMultiValuedDimensionExpression() {
series := 0
multiValuedDimension := ""
for key, values := range query.Dimensions {
@@ -98,18 +131,18 @@ func parseMetricResults(results map[string]*cloudwatch.MetricDataResult, labels
}
for _, value := range query.Dimensions[multiValuedDimension] {
- tags := map[string]string{multiValuedDimension: value}
+ labels := map[string]string{multiValuedDimension: value}
for key, values := range query.Dimensions {
if key != multiValuedDimension && len(values) > 0 {
- tags[key] = values[0]
+ labels[key] = values[0]
}
}
timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, []*time.Time{})
- valueField := data.NewField(data.TimeSeriesValueFieldName, tags, []*float64{})
+ valueField := data.NewField(data.TimeSeriesValueFieldName, labels, []*float64{})
- frameName := formatAlias(query, query.Stats, tags, label)
- valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: frameName})
+ frameName := formatAlias(query, query.Statistic, labels, label)
+ valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: frameName, Links: createDataLinks(deepLink)})
emptyFrame := data.Frame{
Name: frameName,
@@ -118,66 +151,63 @@ func parseMetricResults(results map[string]*cloudwatch.MetricDataResult, labels
valueField,
},
RefID: query.RefId,
+ Meta: createMeta(query),
}
frames = append(frames, &emptyFrame)
}
- } else {
- dims := make([]string, 0, len(query.Dimensions))
- for k := range query.Dimensions {
- dims = append(dims, k)
- }
- sort.Strings(dims)
-
- tags := data.Labels{}
- for _, dim := range dims {
- values := query.Dimensions[dim]
- if len(values) == 1 && values[0] != "*" {
- tags[dim] = values[0]
- } else {
- for _, value := range values {
- if value == label || value == "*" {
- tags[dim] = label
- } else if strings.Contains(label, value) {
- tags[dim] = value
- }
- }
- }
- }
-
- timestamps := []*time.Time{}
- points := []*float64{}
- for j, t := range result.Timestamps {
- if j > 0 {
- expectedTimestamp := result.Timestamps[j-1].Add(time.Duration(query.Period) * time.Second)
- if expectedTimestamp.Before(*t) {
- timestamps = append(timestamps, &expectedTimestamp)
- points = append(points, nil)
- }
- }
- val := result.Values[j]
- timestamps = append(timestamps, t)
- points = append(points, val)
- }
-
- timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, timestamps)
- valueField := data.NewField(data.TimeSeriesValueFieldName, tags, points)
-
- frameName := formatAlias(query, query.Stats, tags, label)
- valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: frameName})
-
- frame := data.Frame{
- Name: frameName,
- Fields: []*data.Field{
- timeField,
- valueField,
- },
- RefID: query.RefId,
- }
- frames = append(frames, &frame)
+ continue
}
+
+ labels := getLabels(label, query)
+ timestamps := []*time.Time{}
+ points := []*float64{}
+ for j, t := range metric.Timestamps {
+ if j > 0 {
+ expectedTimestamp := metric.Timestamps[j-1].Add(time.Duration(query.Period) * time.Second)
+ if expectedTimestamp.Before(*t) {
+ timestamps = append(timestamps, &expectedTimestamp)
+ points = append(points, nil)
+ }
+ }
+ val := metric.Values[j]
+ timestamps = append(timestamps, t)
+ points = append(points, val)
+ }
+
+ timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, timestamps)
+ valueField := data.NewField(data.TimeSeriesValueFieldName, labels, points)
+
+ frameName := formatAlias(query, query.Statistic, labels, label)
+ valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: frameName, Links: createDataLinks(deepLink)})
+
+ frame := data.Frame{
+ Name: frameName,
+ Fields: []*data.Field{
+ timeField,
+ valueField,
+ },
+ RefID: query.RefId,
+ Meta: createMeta(query),
+ }
+
+ if aggregatedResponse.RequestExceededMaxLimit {
+ frame.AppendNotices(data.Notice{
+ Severity: data.NoticeSeverityWarning,
+ Text: "cloudwatch GetMetricData error: Maximum number of allowed metrics exceeded. Your search may have been limited",
+ })
+ }
+
+ if aggregatedResponse.StatusCode != "Complete" {
+ frame.AppendNotices(data.Notice{
+ Severity: data.NoticeSeverityWarning,
+ Text: "cloudwatch GetMetricData error: Too many datapoints requested - your search has been limited. Please try to reduce the time range",
+ })
+ }
+
+ frames = append(frames, &frame)
}
- return frames, partialData, nil
+ return frames, nil
}
func formatAlias(query *cloudWatchQuery, stat string, dimensions map[string]string, label string) string {
@@ -231,3 +261,25 @@ func formatAlias(query *cloudWatchQuery, stat string, dimensions map[string]stri
return string(result)
}
+
+func createDataLinks(link string) []data.DataLink {
+ dataLinks := []data.DataLink{}
+ if link != "" {
+ dataLinks = append(dataLinks, data.DataLink{
+ Title: "View in CloudWatch console",
+ TargetBlank: true,
+ URL: link,
+ })
+ }
+ return dataLinks
+}
+
+func createMeta(query *cloudWatchQuery) *data.FrameMeta {
+ return &data.FrameMeta{
+ ExecutedQueryString: query.UsedExpression,
+ Custom: simplejson.NewFromAny(map[string]interface{}{
+ "period": query.Period,
+ "id": query.Id,
+ }),
+ }
+}
diff --git a/pkg/tsdb/cloudwatch/response_parser_test.go b/pkg/tsdb/cloudwatch/response_parser_test.go
index 3cd1cfcbfda..cdb449d94a6 100644
--- a/pkg/tsdb/cloudwatch/response_parser_test.go
+++ b/pkg/tsdb/cloudwatch/response_parser_test.go
@@ -1,6 +1,8 @@
package cloudwatch
import (
+ "encoding/json"
+ "io/ioutil"
"testing"
"time"
@@ -10,40 +12,86 @@ import (
"github.com/stretchr/testify/require"
)
+func loadGetMetricDataOutputsFromFile() ([]*cloudwatch.GetMetricDataOutput, error) {
+ var getMetricDataOutputs []*cloudwatch.GetMetricDataOutput
+ jsonBody, err := ioutil.ReadFile("./test-data/multiple-outputs.json")
+ if err != nil {
+ return getMetricDataOutputs, err
+ }
+ err = json.Unmarshal(jsonBody, &getMetricDataOutputs)
+ return getMetricDataOutputs, err
+}
+
func TestCloudWatchResponseParser(t *testing.T) {
+ startTime := time.Now()
+ endTime := startTime.Add(2 * time.Hour)
+ t.Run("when aggregating response", func(t *testing.T) {
+ getMetricDataOutputs, err := loadGetMetricDataOutputsFromFile()
+ require.NoError(t, err)
+ aggregatedResponse := aggregateResponse(getMetricDataOutputs)
+ t.Run("response for id a", func(t *testing.T) {
+ idA := "a"
+ t.Run("should have two labels", func(t *testing.T) {
+ assert.Len(t, aggregatedResponse[idA].Labels, 2)
+ assert.Len(t, aggregatedResponse[idA].Metrics, 2)
+ })
+ t.Run("should have points for label1 taken from both getMetricDataOutputs", func(t *testing.T) {
+ assert.Len(t, aggregatedResponse[idA].Metrics["label1"].Values, 10)
+ })
+ t.Run("should have statuscode 'Complete'", func(t *testing.T) {
+ assert.Equal(t, "Complete", aggregatedResponse[idA].StatusCode)
+ })
+ t.Run("should have exceeded request limit", func(t *testing.T) {
+ assert.True(t, aggregatedResponse[idA].RequestExceededMaxLimit)
+ })
+ })
+ t.Run("response for id b", func(t *testing.T) {
+ idB := "b"
+ t.Run("should have statuscode is 'Partial'", func(t *testing.T) {
+ assert.Equal(t, "Partial", aggregatedResponse[idB].StatusCode)
+ })
+ t.Run("should have an arithmetic error and an error message", func(t *testing.T) {
+ assert.True(t, aggregatedResponse[idB].HasArithmeticError)
+ assert.Equal(t, "One or more data-points have been dropped due to non-numeric values (NaN, -Infinite, +Infinite)", aggregatedResponse[idB].ArithmeticErrorMessage)
+ })
+ })
+ })
+
t.Run("Expand dimension value using exact match", func(t *testing.T) {
timestamp := time.Unix(0, 0)
- labels := []string{"lb1", "lb2"}
- mdrs := map[string]*cloudwatch.MetricDataResult{
- "lb1": {
- Id: aws.String("id1"),
- Label: aws.String("lb1"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
+ response := &queryRowResponse{
+ Labels: []string{"lb1", "lb2"},
+ Metrics: map[string]*cloudwatch.MetricDataResult{
+ "lb1": {
+ Id: aws.String("id1"),
+ Label: aws.String("lb1"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
+ "lb2": {
+ Id: aws.String("id2"),
+ Label: aws.String("lb2"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- StatusCode: aws.String("Complete"),
- },
- "lb2": {
- Id: aws.String("id2"),
- Label: aws.String("lb2"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
- },
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
- },
- StatusCode: aws.String("Complete"),
},
}
@@ -56,15 +104,14 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb1", "lb2"},
"TargetGroup": {"tg"},
},
- Stats: "Average",
- Period: 60,
- Alias: "{{LoadBalancer}} Expanded",
+ Statistic: "Average",
+ Period: 60,
+ Alias: "{{LoadBalancer}} Expanded",
}
- frames, partialData, err := parseMetricResults(mdrs, labels, query)
+ frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
frame1 := frames[0]
- assert.False(t, partialData)
assert.Equal(t, "lb1 Expanded", frame1.Name)
assert.Equal(t, "lb1", frame1.Fields[1].Labels["LoadBalancer"])
@@ -75,39 +122,40 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Expand dimension value using substring", func(t *testing.T) {
timestamp := time.Unix(0, 0)
- labels := []string{"lb1 Sum", "lb2 Average"}
- mdrs := map[string]*cloudwatch.MetricDataResult{
- "lb1 Sum": {
- Id: aws.String("id1"),
- Label: aws.String("lb1 Sum"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
+ response := &queryRowResponse{
+ Labels: []string{"lb1 Sum", "lb2 Average"},
+ Metrics: map[string]*cloudwatch.MetricDataResult{
+ "lb1 Sum": {
+ Id: aws.String("id1"),
+ Label: aws.String("lb1 Sum"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
+ "lb2 Average": {
+ Id: aws.String("id2"),
+ Label: aws.String("lb2 Average"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- StatusCode: aws.String("Complete"),
- },
- "lb2 Average": {
- Id: aws.String("id2"),
- Label: aws.String("lb2 Average"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
- },
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
- },
- StatusCode: aws.String("Complete"),
- },
- }
+ }}
query := &cloudWatchQuery{
RefId: "refId1",
@@ -118,15 +166,14 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb1", "lb2"},
"TargetGroup": {"tg"},
},
- Stats: "Average",
- Period: 60,
- Alias: "{{LoadBalancer}} Expanded",
+ Statistic: "Average",
+ Period: 60,
+ Alias: "{{LoadBalancer}} Expanded",
}
- frames, partialData, err := parseMetricResults(mdrs, labels, query)
+ frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
frame1 := frames[0]
- assert.False(t, partialData)
assert.Equal(t, "lb1 Expanded", frame1.Name)
assert.Equal(t, "lb1", frame1.Fields[1].Labels["LoadBalancer"])
@@ -137,37 +184,39 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Expand dimension value using wildcard", func(t *testing.T) {
timestamp := time.Unix(0, 0)
- labels := []string{"lb3", "lb4"}
- mdrs := map[string]*cloudwatch.MetricDataResult{
- "lb3": {
- Id: aws.String("lb3"),
- Label: aws.String("lb3"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
+ response := &queryRowResponse{
+ Labels: []string{"lb3", "lb4"},
+ Metrics: map[string]*cloudwatch.MetricDataResult{
+ "lb3": {
+ Id: aws.String("lb3"),
+ Label: aws.String("lb3"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
+ "lb4": {
+ Id: aws.String("lb4"),
+ Label: aws.String("lb4"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- StatusCode: aws.String("Complete"),
- },
- "lb4": {
- Id: aws.String("lb4"),
- Label: aws.String("lb4"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
- },
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
- },
- StatusCode: aws.String("Complete"),
},
}
@@ -180,35 +229,35 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"*"},
"TargetGroup": {"tg"},
},
- Stats: "Average",
- Period: 60,
- Alias: "{{LoadBalancer}} Expanded",
+ Statistic: "Average",
+ Period: 60,
+ Alias: "{{LoadBalancer}} Expanded",
}
- frames, partialData, err := parseMetricResults(mdrs, labels, query)
+ frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
- assert.False(t, partialData)
assert.Equal(t, "lb3 Expanded", frames[0].Name)
assert.Equal(t, "lb4 Expanded", frames[1].Name)
})
t.Run("Expand dimension value when no values are returned and a multi-valued template variable is used", func(t *testing.T) {
timestamp := time.Unix(0, 0)
- labels := []string{"lb3"}
- mdrs := map[string]*cloudwatch.MetricDataResult{
- "lb3": {
- Id: aws.String("lb3"),
- Label: aws.String("lb3"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
+ response := &queryRowResponse{
+ Labels: []string{"lb3"},
+ Metrics: map[string]*cloudwatch.MetricDataResult{
+ "lb3": {
+ Id: aws.String("lb3"),
+ Label: aws.String("lb3"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{},
+ StatusCode: aws.String("Complete"),
},
- Values: []*float64{},
- StatusCode: aws.String("Complete"),
},
}
-
query := &cloudWatchQuery{
RefId: "refId1",
Region: "us-east-1",
@@ -217,14 +266,13 @@ func TestCloudWatchResponseParser(t *testing.T) {
Dimensions: map[string][]string{
"LoadBalancer": {"lb1", "lb2"},
},
- Stats: "Average",
- Period: 60,
- Alias: "{{LoadBalancer}} Expanded",
+ Statistic: "Average",
+ Period: 60,
+ Alias: "{{LoadBalancer}} Expanded",
}
- frames, partialData, err := parseMetricResults(mdrs, labels, query)
+ frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
- assert.False(t, partialData)
assert.Len(t, frames, 2)
assert.Equal(t, "lb1 Expanded", frames[0].Name)
assert.Equal(t, "lb2 Expanded", frames[1].Name)
@@ -232,18 +280,20 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Expand dimension value when no values are returned and a multi-valued template variable and two single-valued dimensions are used", func(t *testing.T) {
timestamp := time.Unix(0, 0)
- labels := []string{"lb3"}
- mdrs := map[string]*cloudwatch.MetricDataResult{
- "lb3": {
- Id: aws.String("lb3"),
- Label: aws.String("lb3"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
+ response := &queryRowResponse{
+ Labels: []string{"lb3"},
+ Metrics: map[string]*cloudwatch.MetricDataResult{
+ "lb3": {
+ Id: aws.String("lb3"),
+ Label: aws.String("lb3"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{},
+ StatusCode: aws.String("Complete"),
},
- Values: []*float64{},
- StatusCode: aws.String("Complete"),
},
}
@@ -257,14 +307,13 @@ func TestCloudWatchResponseParser(t *testing.T) {
"InstanceType": {"micro"},
"Resource": {"res"},
},
- Stats: "Average",
- Period: 60,
- Alias: "{{LoadBalancer}} Expanded {{InstanceType}} - {{Resource}}",
+ Statistic: "Average",
+ Period: 60,
+ Alias: "{{LoadBalancer}} Expanded {{InstanceType}} - {{Resource}}",
}
- frames, partialData, err := parseMetricResults(mdrs, labels, query)
+ frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
- assert.False(t, partialData)
assert.Len(t, frames, 2)
assert.Equal(t, "lb1 Expanded micro - res", frames[0].Name)
assert.Equal(t, "lb2 Expanded micro - res", frames[1].Name)
@@ -272,22 +321,24 @@ func TestCloudWatchResponseParser(t *testing.T) {
t.Run("Parse cloudwatch response", func(t *testing.T) {
timestamp := time.Unix(0, 0)
- labels := []string{"lb"}
- mdrs := map[string]*cloudwatch.MetricDataResult{
- "lb": {
- Id: aws.String("id1"),
- Label: aws.String("lb"),
- Timestamps: []*time.Time{
- aws.Time(timestamp),
- aws.Time(timestamp.Add(60 * time.Second)),
- aws.Time(timestamp.Add(180 * time.Second)),
+ response := &queryRowResponse{
+ Labels: []string{"lb"},
+ Metrics: map[string]*cloudwatch.MetricDataResult{
+ "lb": {
+ Id: aws.String("id1"),
+ Label: aws.String("lb"),
+ Timestamps: []*time.Time{
+ aws.Time(timestamp),
+ aws.Time(timestamp.Add(60 * time.Second)),
+ aws.Time(timestamp.Add(180 * time.Second)),
+ },
+ Values: []*float64{
+ aws.Float64(10),
+ aws.Float64(20),
+ aws.Float64(30),
+ },
+ StatusCode: aws.String("Complete"),
},
- Values: []*float64{
- aws.Float64(10),
- aws.Float64(20),
- aws.Float64(30),
- },
- StatusCode: aws.String("Complete"),
},
}
@@ -300,15 +351,14 @@ func TestCloudWatchResponseParser(t *testing.T) {
"LoadBalancer": {"lb"},
"TargetGroup": {"tg"},
},
- Stats: "Average",
- Period: 60,
- Alias: "{{namespace}}_{{metric}}_{{stat}}",
+ Statistic: "Average",
+ Period: 60,
+ Alias: "{{namespace}}_{{metric}}_{{stat}}",
}
- frames, partialData, err := parseMetricResults(mdrs, labels, query)
+ frames, err := buildDataFrames(startTime, endTime, *response, query)
require.NoError(t, err)
frame := frames[0]
- assert.False(t, partialData)
assert.Equal(t, "AWS/ApplicationELB_TargetResponseTime_Average", frame.Name)
assert.Equal(t, "Time", frame.Fields[0].Name)
assert.Equal(t, "lb", frame.Fields[1].Labels["LoadBalancer"])
diff --git a/pkg/tsdb/cloudwatch/test-data/multiple-outputs.json b/pkg/tsdb/cloudwatch/test-data/multiple-outputs.json
new file mode 100644
index 00000000000..947b20006a3
--- /dev/null
+++ b/pkg/tsdb/cloudwatch/test-data/multiple-outputs.json
@@ -0,0 +1,96 @@
+[
+ {
+ "Messages": null,
+ "MetricDataResults": [
+ {
+ "Id": "a",
+ "Label": "label1",
+ "Messages": null,
+ "StatusCode": "Complete",
+ "Timestamps": [
+ "2021-01-15T19:44:00Z",
+ "2021-01-15T19:59:00Z",
+ "2021-01-15T20:14:00Z",
+ "2021-01-15T20:29:00Z",
+ "2021-01-15T20:44:00Z"
+ ],
+ "Values": [
+ 0.1333395078879982,
+ 0.244268469636633,
+ 0.15574387947267768,
+ 0.14447563659125626,
+ 0.15519743138527173
+ ]
+ },
+ {
+ "Id": "a",
+ "Label": "label2",
+ "Messages": null,
+ "StatusCode": "Complete",
+ "Timestamps": [
+ "2021-01-15T19:44:00Z"
+ ],
+ "Values": [
+ 0.1333395078879982
+ ]
+ },
+ {
+ "Id": "b",
+ "Label": "label2",
+ "Messages": null,
+ "StatusCode": "Complete",
+ "Timestamps": [
+ "2021-01-15T19:44:00Z"
+ ],
+ "Values": [
+ 0.1333395078879982
+ ]
+ }
+ ],
+ "NextToken": null
+ },
+ {
+ "Messages": [
+ { "Code": "", "Value": null },
+ { "Code": "MaxMetricsExceeded", "Value": null }
+ ],
+ "MetricDataResults": [
+ {
+ "Id": "a",
+ "Label": "label1",
+ "Messages": null,
+ "StatusCode": "Complete",
+ "Timestamps": [
+ "2021-01-15T19:44:00Z",
+ "2021-01-15T19:59:00Z",
+ "2021-01-15T20:14:00Z",
+ "2021-01-15T20:29:00Z",
+ "2021-01-15T20:44:00Z"
+ ],
+ "Values": [
+ 0.1333395078879982,
+ 0.244268469636633,
+ 0.15574387947267768,
+ 0.14447563659125626,
+ 0.15519743138527173
+ ]
+ },
+ {
+ "Id": "b",
+ "Label": "label2",
+ "Messages": [{
+ "Code": "ArithmeticError",
+ "Value": "One or more data-points have been dropped due to non-numeric values (NaN, -Infinite, +Infinite)"
+ }],
+ "StatusCode": "Partial",
+ "Timestamps": [
+ "2021-01-15T19:44:00Z"
+ ],
+ "Values": [
+ 0.1333395078879982
+ ]
+ }
+ ],
+ "NextToken": null
+ }
+]
diff --git a/pkg/tsdb/cloudwatch/time_series_query.go b/pkg/tsdb/cloudwatch/time_series_query.go
index 95c472d2858..25a18373a17 100644
--- a/pkg/tsdb/cloudwatch/time_series_query.go
+++ b/pkg/tsdb/cloudwatch/time_series_query.go
@@ -21,7 +21,6 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, req *ba
if len(req.Queries) == 0 {
return nil, fmt.Errorf("request contains no queries")
}
-
// startTime and endTime are always the same for all queries
startTime := req.Queries[0].TimeRange.From
endTime := req.Queries[0].TimeRange.To
@@ -62,12 +61,7 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, req *ba
return err
}
- queries, err := e.transformRequestQueriesToCloudWatchQueries(requestQueries)
- if err != nil {
- return err
- }
-
- metricDataInput, err := e.buildMetricDataInput(startTime, endTime, queries)
+ metricDataInput, err := e.buildMetricDataInput(startTime, endTime, requestQueries)
if err != nil {
return err
}
@@ -77,22 +71,15 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, req *ba
return err
}
- responses, err := e.parseResponse(mdo, queries)
+ res, err := e.parseResponse(startTime, endTime, mdo, requestQueries)
if err != nil {
return err
}
- res, err := e.transformQueryResponsesToQueryResult(responses, requestQueries, startTime, endTime)
- if err != nil {
- return err
+ for _, responseWrapper := range res {
+ resultChan <- responseWrapper
}
- for refID, queryRes := range res {
- resultChan <- &responseWrapper{
- DataResponse: queryRes,
- RefId: refID,
- }
- }
return nil
})
}
diff --git a/pkg/tsdb/cloudwatch/types.go b/pkg/tsdb/cloudwatch/types.go
index 0aa6939bc6f..bcb55de85df 100644
--- a/pkg/tsdb/cloudwatch/types.go
+++ b/pkg/tsdb/cloudwatch/types.go
@@ -2,37 +2,8 @@ package cloudwatch
import (
"fmt"
-
- "github.com/grafana/grafana-plugin-sdk-go/data"
)
-type requestQuery struct {
- RefId string
- Region string
- Id string
- Namespace string
- MetricName string
- Statistics []*string
- QueryType string
- Expression string
- ReturnData bool
- Dimensions map[string][]string
- ExtendedStatistics []*string
- Period int
- Alias string
- MatchExact bool
-}
-
-type cloudwatchResponse struct {
- DataFrames data.Frames
- Id string
- RefId string
- Expression string
- RequestExceededMaxLimit bool
- PartialData bool
- Period int
-}
-
type queryError struct {
err error
RefID string
@@ -42,11 +13,6 @@ func (e *queryError) Error() string {
return fmt.Sprintf("error parsing query %q, %s", e.RefID, e.err)
}
-type executedQuery struct {
- Expression, ID string
- Period int
-}
-
type cloudWatchLink struct {
View string `json:"view"`
Stacked bool `json:"stacked"`
diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go
index b8582c4af23..92a7a712f36 100644
--- a/pkg/tsdb/elasticsearch/client/client.go
+++ b/pkg/tsdb/elasticsearch/client/client.go
@@ -19,7 +19,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
"golang.org/x/net/context/ctxhttp"
)
@@ -104,13 +104,13 @@ func (c *baseClientImpl) GetTimeField() string {
func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) {
timeInterval := c.ds.TimeInterval
- return tsdb.GetIntervalFrom(queryInterval, timeInterval, 0, 5*time.Second)
+ return intervalv2.GetIntervalFrom(queryInterval, timeInterval, 0, 5*time.Second)
}
type multiRequest struct {
header map[string]interface{}
body interface{}
- interval tsdb.Interval
+ interval intervalv2.Interval
}
func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*response, error) {
diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go
index 3affac77055..d3b2ad8f12f 100644
--- a/pkg/tsdb/elasticsearch/client/client_test.go
+++ b/pkg/tsdb/elasticsearch/client/client_test.go
@@ -13,7 +13,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -319,7 +319,7 @@ func createMultisearchForTest(t *testing.T, c Client) (*MultiSearchRequest, erro
t.Helper()
msb := c.MultiSearch()
- s := msb.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
+ s := msb.Search(intervalv2.Interval{Value: 15 * time.Second, Text: "15s"})
s.Agg().DateHistogram("2", "@timestamp", func(a *DateHistogramAgg, ab AggBuilder) {
a.Interval = "$__interval"
diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go
index a776d3f587f..1c4c0fcc2a9 100644
--- a/pkg/tsdb/elasticsearch/client/models.go
+++ b/pkg/tsdb/elasticsearch/client/models.go
@@ -5,7 +5,7 @@ import (
"net/http"
"github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
)
type response struct {
@@ -32,7 +32,7 @@ type SearchDebugInfo struct {
// SearchRequest represents a search request
type SearchRequest struct {
Index string
- Interval tsdb.Interval
+ Interval intervalv2.Interval
Size int
Sort map[string]interface{}
Query *Query
diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go
index 1b6c35a2935..958120d3faa 100644
--- a/pkg/tsdb/elasticsearch/client/search_request.go
+++ b/pkg/tsdb/elasticsearch/client/search_request.go
@@ -4,13 +4,13 @@ import (
"strings"
"github.com/Masterminds/semver"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
)
// SearchRequestBuilder represents a builder which can build a search request
type SearchRequestBuilder struct {
version *semver.Version
- interval tsdb.Interval
+ interval intervalv2.Interval
index string
size int
sort map[string]interface{}
@@ -20,7 +20,7 @@ type SearchRequestBuilder struct {
}
// NewSearchRequestBuilder create a new search request builder
-func NewSearchRequestBuilder(version *semver.Version, interval tsdb.Interval) *SearchRequestBuilder {
+func NewSearchRequestBuilder(version *semver.Version, interval intervalv2.Interval) *SearchRequestBuilder {
builder := &SearchRequestBuilder{
version: version,
interval: interval,
@@ -129,7 +129,7 @@ func NewMultiSearchRequestBuilder(version *semver.Version) *MultiSearchRequestBu
}
// Search initiates and returns a new search request builder
-func (m *MultiSearchRequestBuilder) Search(interval tsdb.Interval) *SearchRequestBuilder {
+func (m *MultiSearchRequestBuilder) Search(interval intervalv2.Interval) *SearchRequestBuilder {
b := NewSearchRequestBuilder(m.version, interval)
m.requestBuilders = append(m.requestBuilders, b)
return b
diff --git a/pkg/tsdb/elasticsearch/client/search_request_test.go b/pkg/tsdb/elasticsearch/client/search_request_test.go
index eb2c8d2fe59..5472b9fe6db 100644
--- a/pkg/tsdb/elasticsearch/client/search_request_test.go
+++ b/pkg/tsdb/elasticsearch/client/search_request_test.go
@@ -7,7 +7,7 @@ import (
"github.com/Masterminds/semver"
"github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
. "github.com/smartystreets/goconvey/convey"
)
@@ -16,7 +16,7 @@ func TestSearchRequest(t *testing.T) {
timeField := "@timestamp"
Convey("Given new search request builder for es version 5", func() {
version5, _ := semver.NewVersion("5.0.0")
- b := NewSearchRequestBuilder(version5, tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
+ b := NewSearchRequestBuilder(version5, intervalv2.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When building search request", func() {
sr, err := b.Build()
@@ -392,7 +392,7 @@ func TestSearchRequest(t *testing.T) {
Convey("Given new search request builder for es version 2", func() {
version2, _ := semver.NewVersion("2.0.0")
- b := NewSearchRequestBuilder(version2, tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
+ b := NewSearchRequestBuilder(version2, intervalv2.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When adding doc value field", func() {
b.AddDocValueField(timeField)
@@ -452,7 +452,7 @@ func TestMultiSearchRequest(t *testing.T) {
b := NewMultiSearchRequestBuilder(version2)
Convey("When adding one search request", func() {
- b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
+ b.Search(intervalv2.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When building search request should contain one search request", func() {
mr, err := b.Build()
@@ -462,8 +462,8 @@ func TestMultiSearchRequest(t *testing.T) {
})
Convey("When adding two search requests", func() {
- b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
- b.Search(tsdb.Interval{Value: 15 * time.Second, Text: "15s"})
+ b.Search(intervalv2.Interval{Value: 15 * time.Second, Text: "15s"})
+ b.Search(intervalv2.Interval{Value: 15 * time.Second, Text: "15s"})
Convey("When building search request should contain two search requests", func() {
mr, err := b.Build()
diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go
index 4a31ff2115f..6dde9658266 100644
--- a/pkg/tsdb/elasticsearch/elasticsearch.go
+++ b/pkg/tsdb/elasticsearch/elasticsearch.go
@@ -14,15 +14,15 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
- "github.com/grafana/grafana/pkg/tsdb"
es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
)
var eslog = log.New("tsdb.elasticsearch")
type Service struct {
HTTPClientProvider httpclient.Provider
- intervalCalculator tsdb.Calculator
+ intervalCalculator intervalv2.Calculator
im instancemgmt.InstanceManager
}
@@ -49,7 +49,7 @@ func newService(im instancemgmt.InstanceManager, httpClientProvider httpclient.P
return &Service{
im: im,
HTTPClientProvider: httpClientProvider,
- intervalCalculator: tsdb.NewCalculator(),
+ intervalCalculator: intervalv2.NewCalculator(),
}
}
diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go
index 9233ac11de1..bf8ed5c6736 100644
--- a/pkg/tsdb/elasticsearch/models.go
+++ b/pkg/tsdb/elasticsearch/models.go
@@ -6,14 +6,15 @@ import (
// Query represents the time series query model of the datasource
type Query struct {
- TimeField string `json:"timeField"`
- RawQuery string `json:"query"`
- BucketAggs []*BucketAgg `json:"bucketAggs"`
- Metrics []*MetricAgg `json:"metrics"`
- Alias string `json:"alias"`
- Interval string
- IntervalMs int64
- RefID string
+ TimeField string `json:"timeField"`
+ RawQuery string `json:"query"`
+ BucketAggs []*BucketAgg `json:"bucketAggs"`
+ Metrics []*MetricAgg `json:"metrics"`
+ Alias string `json:"alias"`
+ Interval string
+ IntervalMs int64
+ RefID string
+ MaxDataPoints int64
}
// BucketAgg represents a bucket aggregation of the time series query model of the datasource
diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go
index 4caf3cdde4d..345a52eed37 100644
--- a/pkg/tsdb/elasticsearch/time_series_query.go
+++ b/pkg/tsdb/elasticsearch/time_series_query.go
@@ -9,18 +9,18 @@ import (
"github.com/Masterminds/semver"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/tsdb"
es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
)
type timeSeriesQuery struct {
client es.Client
dataQueries []backend.DataQuery
- intervalCalculator tsdb.Calculator
+ intervalCalculator intervalv2.Calculator
}
var newTimeSeriesQuery = func(client es.Client, dataQuery []backend.DataQuery,
- intervalCalculator tsdb.Calculator) *timeSeriesQuery {
+ intervalCalculator intervalv2.Calculator) *timeSeriesQuery {
return &timeSeriesQuery{
client: client,
dataQueries: dataQuery,
@@ -70,12 +70,9 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
if err != nil {
return err
}
- intrvl, err := e.intervalCalculator.Calculate(e.dataQueries[0].TimeRange, minInterval, tsdb.Min)
- if err != nil {
- return err
- }
+ interval := e.intervalCalculator.Calculate(e.dataQueries[0].TimeRange, minInterval, q.MaxDataPoints)
- b := ms.Search(intrvl)
+ b := ms.Search(interval)
b.Size(0)
filters := b.Query().Bool().Filter()
filters.AddDateRangeFilter(e.client.GetTimeField(), to, from, es.DateFormatEpochMS)
@@ -403,13 +400,14 @@ func (p *timeSeriesQueryParser) parse(tsdbQuery []backend.DataQuery) ([]*Query,
interval := model.Get("interval").MustString("")
queries = append(queries, &Query{
- TimeField: timeField,
- RawQuery: rawQuery,
- BucketAggs: bucketAggs,
- Metrics: metrics,
- Alias: alias,
- Interval: interval,
- RefID: q.RefID,
+ TimeField: timeField,
+ RawQuery: rawQuery,
+ BucketAggs: bucketAggs,
+ Metrics: metrics,
+ Alias: alias,
+ Interval: interval,
+ RefID: q.RefID,
+ MaxDataPoints: q.MaxDataPoints,
})
}
diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go
index 9f64bc97cf6..e4a515d372c 100644
--- a/pkg/tsdb/elasticsearch/time_series_query_test.go
+++ b/pkg/tsdb/elasticsearch/time_series_query_test.go
@@ -8,8 +8,8 @@ import (
"github.com/Masterminds/semver"
"github.com/grafana/grafana-plugin-sdk-go/backend"
- "github.com/grafana/grafana/pkg/tsdb"
es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -1150,7 +1150,7 @@ func executeTsdbQuery(c es.Client, body string, from, to time.Time, minInterval
},
},
}
- query := newTimeSeriesQuery(c, dataRequest.Queries, tsdb.NewCalculator(tsdb.CalculatorOptions{MinInterval: minInterval}))
+ query := newTimeSeriesQuery(c, dataRequest.Queries, intervalv2.NewCalculator(intervalv2.CalculatorOptions{MinInterval: minInterval}))
return query.execute()
}
diff --git a/pkg/tsdb/grafanads/grafana.go b/pkg/tsdb/grafanads/grafana.go
new file mode 100644
index 00000000000..4c0a866e46d
--- /dev/null
+++ b/pkg/tsdb/grafanads/grafana.go
@@ -0,0 +1,230 @@
+package grafanads
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/grafana/grafana/pkg/components/securejsondata"
+ "github.com/grafana/grafana/pkg/models"
+
+ "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
+
+ "github.com/grafana/grafana/pkg/plugins/backendplugin"
+
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/data"
+ "github.com/grafana/grafana-plugin-sdk-go/experimental"
+ "github.com/grafana/grafana/pkg/components/simplejson"
+ "github.com/grafana/grafana/pkg/infra/log"
+ "github.com/grafana/grafana/pkg/setting"
+ "github.com/grafana/grafana/pkg/tsdb/testdatasource"
+)
+
+// DatasourceName is the string constant used as the datasource name in requests
+// to identify it as a Grafana DS command.
+const DatasourceName = "-- Grafana --"
+
+// DatasourceID is the fake datasource id used in requests to identify it as a
+// Grafana DS command.
+const DatasourceID = -1
+
+// DatasourceUID is the fake datasource uid used in requests to identify it as a
+// Grafana DS command.
+const DatasourceUID = "grafana"
+
+// Make sure Service implements required interfaces.
+// This is important to do since otherwise we will only get a
+// not implemented error response from plugin at runtime.
+var (
+ _ backend.QueryDataHandler = (*Service)(nil)
+ _ backend.CheckHealthHandler = (*Service)(nil)
+ logger = log.New("tsdb.grafana")
+)
+
+func ProvideService(cfg *setting.Cfg, backendPM backendplugin.Manager) *Service {
+ return newService(cfg.StaticRootPath, backendPM)
+}
+
+func newService(staticRootPath string, backendPM backendplugin.Manager) *Service {
+ s := &Service{
+ staticRootPath: staticRootPath,
+ roots: []string{
+ "testdata",
+ "img/icons",
+ "img/bg",
+ "gazetteer",
+ "upload", // does not exist yet
+ },
+ }
+
+ if err := backendPM.Register("grafana", coreplugin.New(backend.ServeOpts{
+ CheckHealthHandler: s,
+ QueryDataHandler: s,
+ })); err != nil {
+ logger.Error("Failed to register plugin", "error", err)
+ return nil
+ }
+ return s
+}
+
+// Service exists regardless of user settings
+type Service struct {
+ // path to the public folder
+ staticRootPath string
+ roots []string
+}
+
+func DataSourceModel(orgId int64) *models.DataSource {
+ return &models.DataSource{
+ Id: DatasourceID,
+ Uid: DatasourceUID,
+ Name: DatasourceName,
+ Type: "grafana",
+ OrgId: orgId,
+ JsonData: simplejson.New(),
+ SecureJsonData: make(securejsondata.SecureJsonData),
+ }
+}
+
+func (s *Service) QueryData(_ context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+ response := backend.NewQueryDataResponse()
+
+ for _, q := range req.Queries {
+ switch q.QueryType {
+ case queryTypeRandomWalk:
+ response.Responses[q.RefID] = s.doRandomWalk(q)
+ case queryTypeList:
+ response.Responses[q.RefID] = s.doListQuery(q)
+ case queryTypeRead:
+ response.Responses[q.RefID] = s.doReadQuery(q)
+ default:
+ response.Responses[q.RefID] = backend.DataResponse{
+ Error: fmt.Errorf("unknown query type"),
+ }
+ }
+ }
+
+ return response, nil
+}
+
+func (s *Service) CheckHealth(_ context.Context, _ *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
+ return &backend.CheckHealthResult{
+ Status: backend.HealthStatusOk,
+ Message: "OK",
+ }, nil
+}
+
+func (s *Service) publicPath(path string) (string, error) {
+ if strings.Contains(path, "..") {
+ return "", fmt.Errorf("invalid string")
+ }
+
+ ok := false
+ for _, root := range s.roots {
+ if strings.HasPrefix(path, root) {
+ ok = true
+ break
+ }
+ }
+ if !ok {
+ return "", fmt.Errorf("bad root path")
+ }
+ return filepath.Join(s.staticRootPath, path), nil
+}
+
+func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse {
+ q := &listQueryModel{}
+ response := backend.DataResponse{}
+ err := json.Unmarshal(query.JSON, &q)
+ if err != nil {
+ response.Error = err
+ return response
+ }
+
+ if q.Path == "" {
+ count := len(s.roots)
+ names := data.NewFieldFromFieldType(data.FieldTypeString, count)
+ mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
+ names.Name = "name"
+ mtype.Name = "mediaType"
+ for i, f := range s.roots {
+ names.Set(i, f)
+ mtype.Set(i, "directory")
+ }
+ frame := data.NewFrame("", names, mtype)
+ frame.SetMeta(&data.FrameMeta{
+ Type: data.FrameTypeDirectoryListing,
+ })
+ response.Frames = data.Frames{frame}
+ } else {
+ path, err := s.publicPath(q.Path)
+ if err != nil {
+ response.Error = err
+ return response
+ }
+ frame, err := experimental.GetDirectoryFrame(path, false)
+ if err != nil {
+ response.Error = err
+ return response
+ }
+ response.Frames = data.Frames{frame}
+ }
+
+ return response
+}
+
+func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse {
+ q := &listQueryModel{}
+ response := backend.DataResponse{}
+ err := json.Unmarshal(query.JSON, &q)
+ if err != nil {
+ response.Error = err
+ return response
+ }
+
+ if filepath.Ext(q.Path) != ".csv" {
+ response.Error = fmt.Errorf("unsupported file type")
+ return response
+ }
+
+ path, err := s.publicPath(q.Path)
+ if err != nil {
+ response.Error = err
+ return response
+ }
+
+ // Can ignore gosec G304 here, because we check the file pattern above
+ // nolint:gosec
+ fileReader, err := os.Open(path)
+ if err != nil {
+ response.Error = fmt.Errorf("failed to read file")
+ return response
+ }
+
+ defer func() {
+ if err := fileReader.Close(); err != nil {
+ logger.Warn("Failed to close file", "err", err, "path", path)
+ }
+ }()
+
+ frame, err := testdatasource.LoadCsvContent(fileReader, filepath.Base(path))
+ if err != nil {
+ response.Error = err
+ return response
+ }
+ response.Frames = data.Frames{frame}
+ return response
+}
+
+func (s *Service) doRandomWalk(query backend.DataQuery) backend.DataResponse {
+ response := backend.DataResponse{}
+
+ model := simplejson.New()
+ response.Frames = data.Frames{testdatasource.RandomWalk(query, model, 0)}
+
+ return response
+}
diff --git a/pkg/tsdb/grafanads/grafana_test.go b/pkg/tsdb/grafanads/grafana_test.go
new file mode 100644
index 00000000000..3df4df1cc24
--- /dev/null
+++ b/pkg/tsdb/grafanads/grafana_test.go
@@ -0,0 +1,50 @@
+package grafanads
+
+import (
+ "encoding/json"
+ "path"
+ "testing"
+
+ "github.com/grafana/grafana/pkg/plugins/backendplugin"
+
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/experimental"
+ "github.com/stretchr/testify/require"
+)
+
+func asJSON(v interface{}) json.RawMessage {
+ b, _ := json.Marshal(v)
+ return b
+}
+
+func TestReadFolderListing(t *testing.T) {
+ ds := newService("../../../public", &fakeBackendPM{})
+ dr := ds.doListQuery(backend.DataQuery{
+ QueryType: "x",
+ JSON: asJSON(listQueryModel{
+ Path: "testdata",
+ }),
+ })
+ err := experimental.CheckGoldenDataResponse(path.Join("testdata", "list.golden.txt"), &dr, true)
+ require.NoError(t, err)
+}
+
+func TestReadCSVFile(t *testing.T) {
+ ds := newService("../../../public", &fakeBackendPM{})
+ dr := ds.doReadQuery(backend.DataQuery{
+ QueryType: "x",
+ JSON: asJSON(readQueryModel{
+ Path: "testdata/js_libraries.csv",
+ }),
+ })
+ err := experimental.CheckGoldenDataResponse(path.Join("testdata", "jslib.golden.txt"), &dr, true)
+ require.NoError(t, err)
+}
+
+type fakeBackendPM struct {
+ backendplugin.Manager
+}
+
+func (pm *fakeBackendPM) Register(pluginID string, factory backendplugin.PluginFactoryFunc) error {
+ return nil
+}
diff --git a/pkg/tsdb/grafanads/query.go b/pkg/tsdb/grafanads/query.go
new file mode 100644
index 00000000000..533e783aee7
--- /dev/null
+++ b/pkg/tsdb/grafanads/query.go
@@ -0,0 +1,21 @@
+package grafanads
+
+const (
+ // QueryTypeRandomWalk returns a random walk series
+ queryTypeRandomWalk = "randomWalk"
+
+ // QueryTypeList will list the files in a folder
+ queryTypeList = "list"
+
+ // QueryTypeRead will read a file and return it as data frames
+ // currently only .csv files are supported,
+ // other file types will eventually be supported (parquet, etc)
+ queryTypeRead = "read"
+)
+
+type listQueryModel struct {
+ Path string `json:"path"`
+}
+type readQueryModel struct {
+ Path string `json:"path"`
+}
diff --git a/pkg/tsdb/grafanads/testdata/jslib.golden.txt b/pkg/tsdb/grafanads/testdata/jslib.golden.txt
new file mode 100644
index 00000000000..08f17e7b79e
--- /dev/null
+++ b/pkg/tsdb/grafanads/testdata/jslib.golden.txt
@@ -0,0 +1,21 @@
+🌟 This was machine generated. Do not edit. 🌟
+
+Frame[0]
+Name: js_libraries.csv
+Dimensions: 4 Fields by 6 Rows
++-----------------+--------------------+----------------+----------------+
+| Name: Library | Name: Github Stars | Name: Forks | Name: Watchers |
+| Labels: | Labels: | Labels: | Labels: |
+| Type: []*string | Type: []*int64 | Type: []*int64 | Type: []*int64 |
++-----------------+--------------------+----------------+----------------+
+| React.js | 169000 | 34000 | 6700 |
+| Vue | 184000 | 29100 | 6300 |
+| Angular | 73400 | 19300 | 3200 |
+| JQuery | 54900 | 20000 | 3300 |
+| Meteor | 42400 | 5200 | 1700 |
+| Aurelia | 11600 | 684 | 442 |
++-----------------+--------------------+----------------+----------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAKAAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx
diff --git a/pkg/tsdb/grafanads/testdata/list.golden.txt b/pkg/tsdb/grafanads/testdata/list.golden.txt
new file mode 100644
index 00000000000..40a2534d90c
--- /dev/null
+++ b/pkg/tsdb/grafanads/testdata/list.golden.txt
@@ -0,0 +1,24 @@
+🌟 This was machine generated. Do not edit. 🌟
+
+Frame[0] {
+ "type": "directory-listing",
+ "pathSeparator": "/"
+}
+Name:
+Dimensions: 2 Fields by 6 Rows
++--------------------------+------------------+
+| Name: name | Name: media-type |
+| Labels: | Labels: |
+| Type: []string | Type: []string |
++--------------------------+------------------+
+| browser_marketshare.csv | |
+| flight_info_by_state.csv | |
+| gdp_per_capita.csv | |
+| js_libraries.csv | |
+| population_by_state.csv | |
+| weight_height.csv | |
++--------------------------+------------------+
+
+
+====== TEST DATA RESPONSE (arrow base64) ======
+FRAME=QVJST1cxAAD/////uAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAKQAAAADAAAATAAAACgAAAAEAAAA0P7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADw/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAABD///8IAAAAPAAAADAAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAACM////CAAAABQAAAAKAAAAbWVkaWEtdHlwZQAABAAAAG5hbWUAAAAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA/////9gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAADAAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAB4AAAABgAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAACAAAAAAAAAAKAAAAAAAAAAAAAAAAAAAACgAAAAAAAAACAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAACAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAFwAAAC8AAABBAAAAUQAAAGgAAAB5AAAAAAAAAGJyb3dzZXJfbWFya2V0c2hhcmUuY3N2ZmxpZ2h0X2luZm9fYnlfc3RhdGUuY3N2Z2RwX3Blcl9jYXBpdGEuY3N2anNfbGlicmFyaWVzLmNzdnBvcHVsYXRpb25fYnlfc3RhdGUuY3N2d2VpZ2h0X2hlaWdodC5jc3YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAyAEAAAAAAADgAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAACkAAAAAwAAAEwAAAAoAAAABAAAAND+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAAQ////CAAAADwAAAAwAAAAeyJ0eXBlIjoiZGlyZWN0b3J5LWxpc3RpbmciLCJwYXRoU2VwYXJhdG9yIjoiLyJ9AAAAAAQAAABtZXRhAAAAAAIAAAB8AAAABAAAAJ7///8UAAAAQAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAAjP///wgAAAAUAAAACgAAAG1lZGlhLXR5cGUAAAQAAABuYW1lAAAAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABuYW1lAAAAAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAOgBAABBUlJPVzE=
diff --git a/pkg/tsdb/influxdb/influxdb.go b/pkg/tsdb/influxdb/influxdb.go
index bef00d1d120..a774e48df89 100644
--- a/pkg/tsdb/influxdb/influxdb.go
+++ b/pkg/tsdb/influxdb/influxdb.go
@@ -13,7 +13,6 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
@@ -150,17 +149,17 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
}
func (s *Service) getQuery(dsInfo *models.DatasourceInfo, query *backend.QueryDataRequest) (*Query, error) {
- if len(query.Queries) == 0 {
- return nil, fmt.Errorf("query request contains no queries")
- }
+ queryCount := len(query.Queries)
// The model supports multiple queries, but right now this is only used from
// alerting so we only needed to support batch executing 1 query at a time.
- model, err := simplejson.NewJson(query.Queries[0].JSON)
- if err != nil {
- return nil, fmt.Errorf("couldn't unmarshal query")
+ if queryCount != 1 {
+ return nil, fmt.Errorf("query request should contain exactly 1 query, it contains: %d", queryCount)
}
- return s.QueryParser.Parse(model, dsInfo)
+
+ q := query.Queries[0]
+
+ return s.QueryParser.Parse(q)
}
func (s *Service) createRequest(ctx context.Context, dsInfo *models.DatasourceInfo, query string) (*http.Request, error) {
diff --git a/pkg/tsdb/influxdb/model_parser.go b/pkg/tsdb/influxdb/model_parser.go
index 2a0f0f9f4b7..a31071c2093 100644
--- a/pkg/tsdb/influxdb/model_parser.go
+++ b/pkg/tsdb/influxdb/model_parser.go
@@ -1,17 +1,22 @@
package influxdb
import (
+ "fmt"
"strconv"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/tsdb"
- "github.com/grafana/grafana/pkg/tsdb/influxdb/models"
)
type InfluxdbQueryParser struct{}
-func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.DatasourceInfo) (*Query, error) {
+func (qp *InfluxdbQueryParser) Parse(query backend.DataQuery) (*Query, error) {
+ model, err := simplejson.NewJson(query.JSON)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't unmarshal query")
+ }
+
policy := model.Get("policy").MustString("default")
rawQuery := model.Get("query").MustString("")
useRawQuery := model.Get("rawQuery").MustBool(false)
@@ -40,11 +45,13 @@ func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.Data
return nil, err
}
- queryInterval := model.Get("interval").MustString("")
- intervalMS := model.Get("intervalMs").MustInt(0)
- parsedInterval, err := tsdb.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, int64(intervalMS), time.Millisecond*1)
- if err != nil {
- return nil, err
+ interval := query.Interval
+
+ // we make sure it is at least 1 millisecond
+ minInterval := time.Millisecond
+
+ if interval < minInterval {
+ interval = minInterval
}
return &Query{
@@ -55,7 +62,7 @@ func (qp *InfluxdbQueryParser) Parse(model *simplejson.Json, dsInfo *models.Data
Tags: tags,
Selects: selects,
RawQuery: rawQuery,
- Interval: parsedInterval,
+ Interval: interval,
Alias: alias,
UseRawQuery: useRawQuery,
Tz: tz,
diff --git a/pkg/tsdb/influxdb/model_parser_test.go b/pkg/tsdb/influxdb/model_parser_test.go
index e49e2181777..9f45526f207 100644
--- a/pkg/tsdb/influxdb/model_parser_test.go
+++ b/pkg/tsdb/influxdb/model_parser_test.go
@@ -4,14 +4,12 @@ import (
"testing"
"time"
- "github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/tsdb/influxdb/models"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/stretchr/testify/require"
)
func TestInfluxdbQueryParser_Parse(t *testing.T) {
parser := &InfluxdbQueryParser{}
- dsInfo := &models.DatasourceInfo{}
t.Run("can parse influxdb json model", func(t *testing.T) {
json := `
@@ -103,11 +101,13 @@ func TestInfluxdbQueryParser_Parse(t *testing.T) {
]
}
`
- dsInfo.TimeInterval = ">20s"
- modelJSON, err := simplejson.NewJson([]byte(json))
- require.NoError(t, err)
- res, err := parser.Parse(modelJSON, dsInfo)
+ query := backend.DataQuery{
+ JSON: []byte(json),
+ Interval: time.Second * 20,
+ }
+
+ res, err := parser.Parse(query)
require.NoError(t, err)
require.Len(t, res.GroupBy, 3)
require.Len(t, res.Selects, 3)
@@ -162,10 +162,12 @@ func TestInfluxdbQueryParser_Parse(t *testing.T) {
}
`
- modelJSON, err := simplejson.NewJson([]byte(json))
- require.NoError(t, err)
+ query := backend.DataQuery{
+ JSON: []byte(json),
+ Interval: time.Second * 10,
+ }
- res, err := parser.Parse(modelJSON, dsInfo)
+ res, err := parser.Parse(query)
require.NoError(t, err)
require.Equal(t, "RawDummyQuery", res.RawQuery)
require.Len(t, res.GroupBy, 2)
@@ -173,4 +175,23 @@ func TestInfluxdbQueryParser_Parse(t *testing.T) {
require.Empty(t, res.Tags)
require.Equal(t, time.Second*10, res.Interval)
})
+
+ t.Run("will enforce a minInterval of 1 millisecond", func(t *testing.T) {
+ json := `
+ {
+ "query": "RawDummyQuery",
+ "rawQuery": true,
+ "resultFormat": "time_series"
+ }
+ `
+
+ query := backend.DataQuery{
+ JSON: []byte(json),
+ Interval: time.Millisecond * 0,
+ }
+
+ res, err := parser.Parse(query)
+ require.NoError(t, err)
+ require.Equal(t, time.Millisecond*1, res.Interval)
+ })
}
diff --git a/pkg/tsdb/influxdb/query.go b/pkg/tsdb/influxdb/query.go
index 523461f6874..d7de453f73a 100644
--- a/pkg/tsdb/influxdb/query.go
+++ b/pkg/tsdb/influxdb/query.go
@@ -8,7 +8,7 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
)
var (
@@ -29,16 +29,13 @@ func (query *Query) Build(queryContext *backend.QueryDataRequest) (string, error
res += query.renderTz()
}
- calculator := tsdb.NewCalculator(tsdb.CalculatorOptions{})
- i, err := calculator.Calculate(queryContext.Queries[0].TimeRange, query.Interval, tsdb.Min)
- if err != nil {
- return "", err
- }
+ intervalText := intervalv2.FormatDuration(query.Interval)
+ intervalMs := int64(query.Interval / time.Millisecond)
res = strings.ReplaceAll(res, "$timeFilter", query.renderTimeFilter(queryContext))
- res = strings.ReplaceAll(res, "$interval", i.Text)
- res = strings.ReplaceAll(res, "$__interval_ms", strconv.FormatInt(i.Milliseconds(), 10))
- res = strings.ReplaceAll(res, "$__interval", i.Text)
+ res = strings.ReplaceAll(res, "$interval", intervalText)
+ res = strings.ReplaceAll(res, "$__interval_ms", strconv.FormatInt(intervalMs, 10))
+ res = strings.ReplaceAll(res, "$__interval", intervalText)
return res, nil
}
diff --git a/pkg/tsdb/interval/interval.go b/pkg/tsdb/interval/interval.go
index 71d97af68cf..ab271421dd0 100644
--- a/pkg/tsdb/interval/interval.go
+++ b/pkg/tsdb/interval/interval.go
@@ -29,7 +29,7 @@ type intervalCalculator struct {
}
type Calculator interface {
- Calculate(timeRange plugins.DataTimeRange, interval time.Duration, intervalMode string) (Interval, error)
+ Calculate(timeRange plugins.DataTimeRange, interval time.Duration) Interval
CalculateSafeInterval(timeRange plugins.DataTimeRange, resolution int64) Interval
}
@@ -55,29 +55,17 @@ func (i *Interval) Milliseconds() int64 {
return i.Value.Nanoseconds() / int64(time.Millisecond)
}
-func (ic *intervalCalculator) Calculate(timerange plugins.DataTimeRange, interval time.Duration, intervalMode string) (Interval, error) {
+func (ic *intervalCalculator) Calculate(timerange plugins.DataTimeRange, minInterval time.Duration) Interval {
to := timerange.MustGetTo().UnixNano()
from := timerange.MustGetFrom().UnixNano()
calculatedInterval := time.Duration((to - from) / DefaultRes)
- switch intervalMode {
- case "min":
- if calculatedInterval < interval {
- return Interval{Text: FormatDuration(interval), Value: interval}, nil
- }
- case "max":
- if calculatedInterval > interval {
- return Interval{Text: FormatDuration(interval), Value: interval}, nil
- }
- case "exact":
- return Interval{Text: FormatDuration(interval), Value: interval}, nil
-
- default:
- return Interval{}, fmt.Errorf("unrecognized intervalMode: %v", intervalMode)
+ if calculatedInterval < minInterval {
+ return Interval{Text: FormatDuration(minInterval), Value: minInterval}
}
rounded := roundInterval(calculatedInterval)
- return Interval{Text: FormatDuration(rounded), Value: rounded}, nil
+ return Interval{Text: FormatDuration(rounded), Value: rounded}
}
func (ic *intervalCalculator) CalculateSafeInterval(timerange plugins.DataTimeRange, safeRes int64) Interval {
diff --git a/pkg/tsdb/interval/interval_test.go b/pkg/tsdb/interval/interval_test.go
index c2c6fe8d90e..4f5501aeeb2 100644
--- a/pkg/tsdb/interval/interval_test.go
+++ b/pkg/tsdb/interval/interval_test.go
@@ -15,29 +15,19 @@ func TestIntervalCalculator_Calculate(t *testing.T) {
calculator := NewCalculator(CalculatorOptions{})
testCases := []struct {
- name string
- timeRange plugins.DataTimeRange
- intervalMode string
- expected string
+ name string
+ timeRange plugins.DataTimeRange
+ expected string
}{
- {"from 5m to now", plugins.NewDataTimeRange("5m", "now"), "min", "200ms"},
- {"from 5m to now", plugins.NewDataTimeRange("5m", "now"), "exact", "1ms"},
- {"from 5m to now", plugins.NewDataTimeRange("5m", "now"), "max", "1ms"},
- {"from 15m to now", plugins.NewDataTimeRange("15m", "now"), "min", "500ms"},
- {"from 15m to now", plugins.NewDataTimeRange("15m", "now"), "max", "1ms"},
- {"from 15m to now", plugins.NewDataTimeRange("15m", "now"), "exact", "1ms"},
- {"from 30m to now", plugins.NewDataTimeRange("30m", "now"), "min", "1s"},
- {"from 30m to now", plugins.NewDataTimeRange("30m", "now"), "max", "1ms"},
- {"from 30m to now", plugins.NewDataTimeRange("30m", "now"), "exact", "1ms"},
- {"from 24h to now", plugins.NewDataTimeRange("24h", "now"), "min", "1m"},
- {"from 24h to now", plugins.NewDataTimeRange("24h", "now"), "max", "1ms"},
- {"from 24h to now", plugins.NewDataTimeRange("24h", "now"), "exact", "1ms"},
+ {"from 5m to now", plugins.NewDataTimeRange("5m", "now"), "200ms"},
+ {"from 15m to now", plugins.NewDataTimeRange("15m", "now"), "500ms"},
+ {"from 30m to now", plugins.NewDataTimeRange("30m", "now"), "1s"},
+ {"from 1h to now", plugins.NewDataTimeRange("1h", "now"), "2s"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
- interval, err := calculator.Calculate(tc.timeRange, time.Millisecond*1, tc.intervalMode)
- require.Nil(t, err)
+ interval := calculator.Calculate(tc.timeRange, time.Millisecond*1)
assert.Equal(t, tc.expected, interval.Text)
})
}
diff --git a/pkg/tsdb/calculator.go b/pkg/tsdb/intervalv2/intervalv2.go
similarity index 83%
rename from pkg/tsdb/calculator.go
rename to pkg/tsdb/intervalv2/intervalv2.go
index 262fb1452df..14014cd1306 100644
--- a/pkg/tsdb/calculator.go
+++ b/pkg/tsdb/intervalv2/intervalv2.go
@@ -1,4 +1,4 @@
-package tsdb
+package intervalv2
import (
"fmt"
@@ -7,7 +7,6 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
- "github.com/grafana/grafana/pkg/components/gtime"
"github.com/grafana/grafana/pkg/tsdb/interval"
)
@@ -18,14 +17,6 @@ var (
day = time.Hour * 24
)
-type IntervalMode string
-
-const (
- Min IntervalMode = "min"
- Max IntervalMode = "max"
- Exact IntervalMode = "exact"
-)
-
type Interval struct {
Text string
Value time.Duration
@@ -36,7 +27,7 @@ type intervalCalculator struct {
}
type Calculator interface {
- Calculate(timerange backend.TimeRange, minInterval time.Duration, intervalMode IntervalMode) (Interval, error)
+ Calculate(timerange backend.TimeRange, minInterval time.Duration, maxDataPoints int64) Interval
CalculateSafeInterval(timerange backend.TimeRange, resolution int64) Interval
}
@@ -62,29 +53,23 @@ func (i *Interval) Milliseconds() int64 {
return i.Value.Nanoseconds() / int64(time.Millisecond)
}
-func (ic *intervalCalculator) Calculate(timerange backend.TimeRange, intrvl time.Duration, intervalMode IntervalMode) (Interval, error) {
+func (ic *intervalCalculator) Calculate(timerange backend.TimeRange, minInterval time.Duration, maxDataPoints int64) Interval {
to := timerange.To.UnixNano()
from := timerange.From.UnixNano()
- calculatedIntrvl := time.Duration((to - from) / defaultRes)
-
- switch intervalMode {
- case Min:
- if calculatedIntrvl < intrvl {
- return Interval{Text: interval.FormatDuration(intrvl), Value: intrvl}, nil
- }
- case Max:
- if calculatedIntrvl > intrvl {
- return Interval{Text: interval.FormatDuration(intrvl), Value: intrvl}, nil
- }
- case Exact:
- return Interval{Text: interval.FormatDuration(intrvl), Value: intrvl}, nil
-
- default:
- return Interval{}, fmt.Errorf("unrecognized intervalMode: %v", intervalMode)
+ resolution := maxDataPoints
+ if resolution == 0 {
+ resolution = defaultRes
}
- rounded := roundInterval(calculatedIntrvl)
- return Interval{Text: interval.FormatDuration(rounded), Value: rounded}, nil
+ calculatedInterval := time.Duration((to - from) / resolution)
+
+ if calculatedInterval < minInterval {
+ return Interval{Text: interval.FormatDuration(minInterval), Value: minInterval}
+ }
+
+ rounded := roundInterval(calculatedInterval)
+
+ return Interval{Text: interval.FormatDuration(rounded), Value: rounded}
}
func (ic *intervalCalculator) CalculateSafeInterval(timerange backend.TimeRange, safeRes int64) Interval {
@@ -101,27 +86,41 @@ func (ic *intervalCalculator) CalculateSafeInterval(timerange backend.TimeRange,
// queryInterval is the string representation of query interval (min interval), e.g. "10ms" or "10s".
// queryIntervalMS is a pre-calculated numeric representation of the query interval in milliseconds.
func GetIntervalFrom(dsInterval, queryInterval string, queryIntervalMS int64, defaultInterval time.Duration) (time.Duration, error) {
- if queryInterval == "" {
+ // Apparently we are setting default value of queryInterval to 0s now
+ interval := queryInterval
+ if interval == "0s" {
+ interval = ""
+ }
+ if interval == "" {
if queryIntervalMS != 0 {
return time.Duration(queryIntervalMS) * time.Millisecond, nil
}
}
- interval := queryInterval
- if queryInterval == "" && dsInterval != "" {
+ if interval == "" && dsInterval != "" {
interval = dsInterval
}
if interval == "" {
return defaultInterval, nil
}
- interval = strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1)
- isPureNum, err := regexp.MatchString(`^\d+$`, interval)
+
+ parsedInterval, err := ParseIntervalStringToTimeDuration(interval)
+ if err != nil {
+ return time.Duration(0), err
+ }
+
+ return parsedInterval, nil
+}
+
+func ParseIntervalStringToTimeDuration(interval string) (time.Duration, error) {
+ formattedInterval := strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1)
+ isPureNum, err := regexp.MatchString(`^\d+$`, formattedInterval)
if err != nil {
return time.Duration(0), err
}
if isPureNum {
- interval += "s"
+ formattedInterval += "s"
}
- parsedInterval, err := gtime.ParseDuration(interval)
+ parsedInterval, err := time.ParseDuration(formattedInterval)
if err != nil {
return time.Duration(0), err
}
diff --git a/pkg/tsdb/calculator_test.go b/pkg/tsdb/intervalv2/intervalv2_test.go
similarity index 67%
rename from pkg/tsdb/calculator_test.go
rename to pkg/tsdb/intervalv2/intervalv2_test.go
index 5baec34b7e2..c5c8f5db858 100644
--- a/pkg/tsdb/calculator_test.go
+++ b/pkg/tsdb/intervalv2/intervalv2_test.go
@@ -1,4 +1,4 @@
-package tsdb
+package intervalv2
import (
"testing"
@@ -7,7 +7,6 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
)
func TestIntervalCalculator_Calculate(t *testing.T) {
@@ -16,29 +15,24 @@ func TestIntervalCalculator_Calculate(t *testing.T) {
timeNow := time.Now()
testCases := []struct {
- name string
- timeRange backend.TimeRange
- intervalMode IntervalMode
- expected string
+ name string
+ timeRange backend.TimeRange
+ resolution int64
+ expected string
}{
- {"from 5m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(5 * time.Minute)}, Min, "200ms"},
- {"from 5m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(5 * time.Minute)}, Max, "1ms"},
- {"from 5m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(5 * time.Minute)}, Exact, "1ms"},
- {"from 15m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(15 * time.Minute)}, Min, "500ms"},
- {"from 15m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(15 * time.Minute)}, Max, "1ms"},
- {"from 15m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(15 * time.Minute)}, Exact, "1ms"},
- {"from 30m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(30 * time.Minute)}, Min, "1s"},
- {"from 30m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(30 * time.Minute)}, Max, "1ms"},
- {"from 30m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(30 * time.Minute)}, Exact, "1ms"},
- {"from 1h to now", backend.TimeRange{From: timeNow, To: timeNow.Add(1440 * time.Minute)}, Min, "1m"},
- {"from 1h to now", backend.TimeRange{From: timeNow, To: timeNow.Add(1440 * time.Minute)}, Max, "1ms"},
- {"from 1h to now", backend.TimeRange{From: timeNow, To: timeNow.Add(1440 * time.Minute)}, Exact, "1ms"},
+ {"from 5m to now and default resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(5 * time.Minute)}, 0, "200ms"},
+ {"from 5m to now and 500 resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(5 * time.Minute)}, 500, "500ms"},
+ {"from 15m to now and default resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(15 * time.Minute)}, 0, "500ms"},
+ {"from 15m to now and 100 resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(15 * time.Minute)}, 100, "10s"},
+ {"from 30m to now and default resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(30 * time.Minute)}, 0, "1s"},
+ {"from 30m to now and 3000 resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(30 * time.Minute)}, 3000, "500ms"},
+ {"from 1h to now and default resolution", backend.TimeRange{From: timeNow, To: timeNow.Add(60 * time.Minute)}, 0, "2s"},
+ {"from 1h to now and 1000 resoluion", backend.TimeRange{From: timeNow, To: timeNow.Add(60 * time.Minute)}, 1000, "5s"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
- interval, err := calculator.Calculate(tc.timeRange, time.Millisecond*1, tc.intervalMode)
- require.Nil(t, err)
+ interval := calculator.Calculate(tc.timeRange, time.Millisecond*1, tc.resolution)
assert.Equal(t, tc.expected, interval.Text)
})
}
diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go
index 6acb97d07c7..24e9dd86212 100644
--- a/pkg/tsdb/loki/loki.go
+++ b/pkg/tsdb/loki/loki.go
@@ -18,7 +18,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/grafana/loki/pkg/logcli/client"
"github.com/grafana/loki/pkg/loghttp"
"github.com/grafana/loki/pkg/logproto"
@@ -29,7 +29,7 @@ import (
)
type Service struct {
- intervalCalculator tsdb.Calculator
+ intervalCalculator intervalv2.Calculator
im instancemgmt.InstanceManager
plog log.Logger
}
@@ -38,7 +38,7 @@ func ProvideService(httpClientProvider httpclient.Provider, manager backendplugi
im := datasource.NewInstanceManager(newInstanceSettings(httpClientProvider))
s := &Service{
im: im,
- intervalCalculator: tsdb.NewCalculator(),
+ intervalCalculator: intervalv2.NewCalculator(),
plog: log.New("tsdb.loki"),
}
@@ -195,15 +195,12 @@ func (s *Service) parseQuery(dsInfo *datasourceInfo, queryContext *backend.Query
start := query.TimeRange.From
end := query.TimeRange.To
- dsInterval, err := tsdb.GetIntervalFrom(dsInfo.TimeInterval, model.Interval, int64(model.IntervalMS), time.Second)
+ dsInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, model.Interval, int64(model.IntervalMS), time.Second)
if err != nil {
return nil, fmt.Errorf("failed to parse Interval: %v", err)
}
- interval, err := s.intervalCalculator.Calculate(query.TimeRange, dsInterval, tsdb.Min)
- if err != nil {
- return nil, err
- }
+ interval := s.intervalCalculator.Calculate(query.TimeRange, dsInterval, query.MaxDataPoints)
var resolution int64 = 1
if model.Resolution >= 1 && model.Resolution <= 5 || model.Resolution == 10 {
diff --git a/pkg/tsdb/loki/loki_test.go b/pkg/tsdb/loki/loki_test.go
index cd95a36b7a6..d7632a29c11 100644
--- a/pkg/tsdb/loki/loki_test.go
+++ b/pkg/tsdb/loki/loki_test.go
@@ -8,7 +8,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/grafana/loki/pkg/loghttp"
p "github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
@@ -62,7 +62,7 @@ func TestLoki(t *testing.T) {
}
service := &Service{
intervalCalculator: mockCalculator{
- interval: tsdb.Interval{
+ interval: intervalv2.Interval{
Value: time.Second * 30,
},
},
@@ -93,7 +93,7 @@ func TestLoki(t *testing.T) {
}
service := &Service{
intervalCalculator: mockCalculator{
- interval: tsdb.Interval{
+ interval: intervalv2.Interval{
Value: time.Minute * 2,
},
},
@@ -105,7 +105,7 @@ func TestLoki(t *testing.T) {
service = &Service{
intervalCalculator: mockCalculator{
- interval: tsdb.Interval{
+ interval: intervalv2.Interval{
Value: time.Second * 2,
},
},
@@ -175,13 +175,13 @@ func TestParseResponse(t *testing.T) {
}
type mockCalculator struct {
- interval tsdb.Interval
+ interval intervalv2.Interval
}
-func (m mockCalculator) Calculate(timerange backend.TimeRange, minInterval time.Duration, intervalMode tsdb.IntervalMode) (tsdb.Interval, error) {
- return m.interval, nil
+func (m mockCalculator) Calculate(timerange backend.TimeRange, minInterval time.Duration, maxDataPoints int64) intervalv2.Interval {
+ return m.interval
}
-func (m mockCalculator) CalculateSafeInterval(timerange backend.TimeRange, resolution int64) tsdb.Interval {
+func (m mockCalculator) CalculateSafeInterval(timerange backend.TimeRange, resolution int64) intervalv2.Interval {
return m.interval
}
diff --git a/pkg/tsdb/mssql/macros.go b/pkg/tsdb/mssql/macros.go
index 812cf45bf5d..e03db2be0e6 100644
--- a/pkg/tsdb/mssql/macros.go
+++ b/pkg/tsdb/mssql/macros.go
@@ -6,8 +6,8 @@ import (
"strings"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/gtime"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
@@ -22,7 +22,7 @@ func newMssqlMacroEngine() sqleng.SQLMacroEngine {
return &msSQLMacroEngine{SQLMacroEngineBase: sqleng.NewSQLMacroEngineBase()}
}
-func (m *msSQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange,
+func (m *msSQLMacroEngine) Interpolate(query *backend.DataQuery, timeRange backend.TimeRange,
sql string) (string, error) {
// TODO: Return any error
rExp, _ := regexp.Compile(sExpr)
@@ -48,7 +48,7 @@ func (m *msSQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plu
return sql, nil
}
-func (m *msSQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query plugins.DataSubQuery, name string, args []string) (string, error) {
+func (m *msSQLMacroEngine) evaluateMacro(timeRange backend.TimeRange, query *backend.DataQuery, name string, args []string) (string, error) {
switch name {
case "__time":
if len(args) == 0 {
@@ -65,11 +65,11 @@ func (m *msSQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], timeRange.GetFromAsTimeUTC().Format(time.RFC3339), timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
+ return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], timeRange.From.UTC().Format(time.RFC3339), timeRange.To.UTC().Format(time.RFC3339)), nil
case "__timeFrom":
- return fmt.Sprintf("'%s'", timeRange.GetFromAsTimeUTC().Format(time.RFC3339)), nil
+ return fmt.Sprintf("'%s'", timeRange.From.UTC().Format(time.RFC3339)), nil
case "__timeTo":
- return fmt.Sprintf("'%s'", timeRange.GetToAsTimeUTC().Format(time.RFC3339)), nil
+ return fmt.Sprintf("'%s'", timeRange.To.UTC().Format(time.RFC3339)), nil
case "__timeGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval", name)
@@ -95,16 +95,16 @@ func (m *msSQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsSecondsEpoch(), args[0], timeRange.GetToAsSecondsEpoch()), nil
+ return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().Unix(), args[0], timeRange.To.UTC().Unix()), nil
case "__unixEpochNanoFilter":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsTimeUTC().UnixNano(), args[0], timeRange.GetToAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().UnixNano(), args[0], timeRange.To.UTC().UnixNano()), nil
case "__unixEpochNanoFrom":
- return fmt.Sprintf("%d", timeRange.GetFromAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%d", timeRange.From.UTC().UnixNano()), nil
case "__unixEpochNanoTo":
- return fmt.Sprintf("%d", timeRange.GetToAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%d", timeRange.To.UTC().UnixNano()), nil
case "__unixEpochGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name)
diff --git a/pkg/tsdb/mssql/macros_test.go b/pkg/tsdb/mssql/macros_test.go
index 9ab9a68e045..2b6e568cb76 100644
--- a/pkg/tsdb/mssql/macros_test.go
+++ b/pkg/tsdb/mssql/macros_test.go
@@ -2,14 +2,12 @@ package mssql
import (
"fmt"
- "strconv"
"sync"
"testing"
"time"
- "github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/plugins"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
. "github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/require"
)
@@ -17,16 +15,16 @@ import (
func TestMacroEngine(t *testing.T) {
Convey("MacroEngine", t, func() {
engine := &msSQLMacroEngine{}
- query := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query := &backend.DataQuery{
+ JSON: []byte("{}"),
}
- dfltTimeRange := plugins.DataTimeRange{}
+ dfltTimeRange := backend.TimeRange{}
Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", Now: to, To: "now"}
+ timeRange := backend.TimeRange{From: from, To: to}
Convey("interpolate __time function", func() {
sql, err := engine.Interpolate(query, dfltTimeRange, "select $__time(time_column)")
@@ -92,41 +90,26 @@ func TestMacroEngine(t *testing.T) {
Convey("interpolate __timeGroup function with fill (value = NULL)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)")
-
- fill := query.Model.Get("fill").MustBool()
- fillMode := query.Model.Get("fillMode").MustString()
- fillInterval := query.Model.Get("fillInterval").MustInt()
-
So(err, ShouldBeNil)
- So(fill, ShouldBeTrue)
- So(fillMode, ShouldEqual, "null")
- So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
+ queryJson, err := query.JSON.MarshalJSON()
+ So(err, ShouldBeNil)
+ So(string(queryJson), ShouldEqual, `{"fill":true,"fillInterval":300,"fillMode":"null"}`)
})
Convey("interpolate __timeGroup function with fill (value = previous)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', previous)")
-
- fill := query.Model.Get("fill").MustBool()
- fillMode := query.Model.Get("fillMode").MustString()
- fillInterval := query.Model.Get("fillInterval").MustInt()
-
So(err, ShouldBeNil)
- So(fill, ShouldBeTrue)
- So(fillMode, ShouldEqual, "previous")
- So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
+ queryJson, err := query.JSON.MarshalJSON()
+ So(err, ShouldBeNil)
+ So(string(queryJson), ShouldEqual, `{"fill":true,"fillInterval":300,"fillMode":"previous"}`)
})
Convey("interpolate __timeGroup function with fill (value = float)", func() {
_, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)")
-
- fill := query.Model.Get("fill").MustBool()
- fillValue := query.Model.Get("fillValue").MustFloat64()
- fillInterval := query.Model.Get("fillInterval").MustInt()
-
So(err, ShouldBeNil)
- So(fill, ShouldBeTrue)
- So(fillValue, ShouldEqual, 1.5)
- So(fillInterval, ShouldEqual, 5*time.Minute.Seconds())
+ queryJson, err := query.JSON.MarshalJSON()
+ So(err, ShouldBeNil)
+ So(string(queryJson), ShouldEqual, `{"fill":true,"fillInterval":300,"fillMode":"value","fillValue":1.5}`)
})
Convey("interpolate __unixEpochFilter function", func() {
@@ -170,9 +153,10 @@ func TestMacroEngine(t *testing.T) {
Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10),
- strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -199,9 +183,10 @@ func TestMacroEngine(t *testing.T) {
Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10),
- strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -229,27 +214,30 @@ func TestMacroEngine(t *testing.T) {
func TestMacroEngineConcurrency(t *testing.T) {
engine := newMssqlMacroEngine()
- query1 := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query1 := backend.DataQuery{
+ JSON: []byte{},
}
- query2 := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query2 := backend.DataQuery{
+ JSON: []byte{},
}
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
var wg sync.WaitGroup
wg.Add(2)
- go func(query plugins.DataSubQuery) {
+ go func(query backend.DataQuery) {
defer wg.Done()
- _, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+ _, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
}(query1)
- go func(query plugins.DataSubQuery) {
- _, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+ go func(query backend.DataQuery) {
+ _, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
defer wg.Done()
}(query2)
diff --git a/pkg/tsdb/mssql/mssql.go b/pkg/tsdb/mssql/mssql.go
index 050c50aceab..ead5ff3ed71 100644
--- a/pkg/tsdb/mssql/mssql.go
+++ b/pkg/tsdb/mssql/mssql.go
@@ -1,6 +1,8 @@
package mssql
import (
+ "context"
+ "encoding/json"
"fmt"
"net/url"
"reflect"
@@ -8,43 +10,104 @@ import (
"strconv"
"strings"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
+ "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
+ "github.com/grafana/grafana/pkg/plugins/backendplugin"
+ "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
mssql "github.com/denisenkom/go-mssqldb"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
var logger = log.New("tsdb.mssql")
-//nolint: staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(datasource *models.DataSource) (plugins.DataPlugin, error) {
- cnnstr, err := generateConnectionString(datasource)
+type Service struct {
+ im instancemgmt.InstanceManager
+}
+
+func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager) (*Service, error) {
+ s := &Service{
+ im: datasource.NewInstanceManager(newInstanceSettings(cfg)),
+ }
+ factory := coreplugin.New(backend.ServeOpts{
+ QueryDataHandler: s,
+ })
+
+ if err := manager.Register("mssql", factory); err != nil {
+ logger.Error("Failed to register plugin", "error", err)
+ }
+ return s, nil
+}
+
+func (s *Service) getDataSourceHandler(pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) {
+ i, err := s.im.Get(pluginCtx)
if err != nil {
return nil, err
}
- // TODO: Don't use global
- if setting.Env == setting.Dev {
- logger.Debug("getEngine", "connection", cnnstr)
- }
+ instance := i.(*sqleng.DataSourceHandler)
+ return instance, nil
+}
- config := sqleng.DataPluginConfiguration{
- DriverName: "mssql",
- ConnectionString: cnnstr,
- Datasource: datasource,
- MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
+func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+ dsHandler, err := s.getDataSourceHandler(req.PluginContext)
+ if err != nil {
+ return nil, err
}
+ return dsHandler.QueryData(ctx, req)
+}
- queryResultTransformer := mssqlQueryResultTransformer{
- log: logger,
+func newInstanceSettings(cfg *setting.Cfg) datasource.InstanceFactoryFunc {
+ return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+ jsonData := sqleng.JsonData{
+ MaxOpenConns: 0,
+ MaxIdleConns: 2,
+ ConnMaxLifetime: 14400,
+ Encrypt: "false",
+ }
+
+ err := json.Unmarshal(settings.JSONData, &jsonData)
+ if err != nil {
+ return nil, fmt.Errorf("error reading settings: %w", err)
+ }
+ dsInfo := sqleng.DataSourceInfo{
+ JsonData: jsonData,
+ URL: settings.URL,
+ User: settings.User,
+ Database: settings.Database,
+ ID: settings.ID,
+ Updated: settings.Updated,
+ UID: settings.UID,
+ DecryptedSecureJSONData: settings.DecryptedSecureJSONData,
+ }
+ cnnstr, err := generateConnectionString(dsInfo)
+ if err != nil {
+ return nil, err
+ }
+
+ if cfg.Env == setting.Dev {
+ logger.Debug("getEngine", "connection", cnnstr)
+ }
+
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "mssql",
+ ConnectionString: cnnstr,
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
+ RowLimit: cfg.DataProxyRowLimit,
+ }
+
+ queryResultTransformer := mssqlQueryResultTransformer{
+ log: logger,
+ }
+
+ return sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
}
-
- return sqleng.NewDataPlugin(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
}
// ParseURL tries to parse an MSSQL URL string into a URL object.
@@ -68,11 +131,11 @@ func ParseURL(u string) (*url.URL, error) {
}, nil
}
-func generateConnectionString(dataSource *models.DataSource) (string, error) {
+func generateConnectionString(dsInfo sqleng.DataSourceInfo) (string, error) {
const dfltPort = "0"
var addr util.NetworkAddress
- if dataSource.Url != "" {
- u, err := ParseURL(dataSource.Url)
+ if dsInfo.URL != "" {
+ u, err := ParseURL(dsInfo.URL)
if err != nil {
return "", err
}
@@ -88,26 +151,30 @@ func generateConnectionString(dataSource *models.DataSource) (string, error) {
}
args := []interface{}{
- "url", dataSource.Url, "host", addr.Host,
+ "url", dsInfo.URL, "host", addr.Host,
}
if addr.Port != "0" {
args = append(args, "port", addr.Port)
}
logger.Debug("Generating connection string", args...)
- encrypt := dataSource.JsonData.Get("encrypt").MustString("false")
connStr := fmt.Sprintf("server=%s;database=%s;user id=%s;password=%s;",
addr.Host,
- dataSource.Database,
- dataSource.User,
- dataSource.DecryptedPassword(),
+ dsInfo.Database,
+ dsInfo.User,
+ dsInfo.DecryptedSecureJSONData["password"],
)
// Port number 0 means to determine the port automatically, so we can let the driver choose
if addr.Port != "0" {
connStr += fmt.Sprintf("port=%s;", addr.Port)
}
- if encrypt != "false" {
- connStr += fmt.Sprintf("encrypt=%s;", encrypt)
+
+ if dsInfo.JsonData.Encrypt == "" {
+ dsInfo.JsonData.Encrypt = "false"
+ }
+
+ if dsInfo.JsonData.Encrypt != "false" {
+ connStr += fmt.Sprintf("encrypt=%s;", dsInfo.JsonData.Encrypt)
}
return connStr, nil
}
diff --git a/pkg/tsdb/mssql/mssql_test.go b/pkg/tsdb/mssql/mssql_test.go
index b13ff27f4ea..31ac7602d11 100644
--- a/pkg/tsdb/mssql/mssql_test.go
+++ b/pkg/tsdb/mssql/mssql_test.go
@@ -8,12 +8,9 @@ import (
"testing"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
- "github.com/grafana/grafana/pkg/components/securejsondata"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
@@ -51,10 +48,18 @@ func TestMSSQL(t *testing.T) {
return x, nil
}
- endpoint, err := NewExecutor(&models.DataSource{
- JsonData: simplejson.New(),
- SecureJsonData: securejsondata.SecureJsonData{},
- })
+ queryResultTransformer := mssqlQueryResultTransformer{
+ log: logger,
+ }
+ dsInfo := sqleng.DataSourceInfo{}
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "mssql",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
+ RowLimit: 1000000,
+ }
+ endpoint, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
require.NoError(t, err)
sess := x.NewSession()
@@ -127,31 +132,27 @@ func TestMSSQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a table query should map MSSQL column types to Go types", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": "SELECT * FROM mssql_types",
- "format": "table",
- }),
+ JSON: []byte(`{"rawSql": "SELECT * FROM mssql_types", "format": "table"}`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), &query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NotNil(t, queryResult)
require.NoError(t, queryResult.Error)
- require.NotNil(t, queryResult.Dataframes)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 24, len(frames[0].Fields))
- require.Equal(t, true, frames[0].Fields[0].At(0).(bool))
+ require.Equal(t, true, *frames[0].Fields[0].At(0).(*bool))
require.Equal(t, int64(5), *frames[0].Fields[1].At(0).(*int64))
require.Equal(t, int64(20020), *frames[0].Fields[2].At(0).(*int64))
require.Equal(t, int64(980300), *frames[0].Fields[3].At(0).(*int64))
@@ -224,24 +225,23 @@ func TestMSSQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using timeGroup", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, '5m') ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"}`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
// without fill this should result in 4 buckets
require.Equal(t, 4, frames[0].Fields[0].Len())
@@ -268,28 +268,28 @@ func TestMSSQL(t *testing.T) {
})
t.Run("When doing a metric query using timeGroup with NULL fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, '5m') ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 7, frames[0].Fields[0].Len())
@@ -322,56 +322,54 @@ func TestMSSQL(t *testing.T) {
t.Run("When doing a metric query using timeGroup and $__interval", func(t *testing.T) {
t.Run("Should replace $__interval", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{},
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, $__interval) AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, $__interval) ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"}`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(30 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(30*time.Minute).Unix()*1000),
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, "SELECT FLOOR(DATEDIFF(second, '1970-01-01', time)/60)*60 AS time, avg(value) as value FROM metric GROUP BY FLOOR(DATEDIFF(second, '1970-01-01', time)/60)*60 ORDER BY 1", frames[0].Meta.ExecutedQueryString)
})
})
t.Run("When doing a metric query using timeGroup with float fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY $__timeGroup(time, '5m') ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 7, frames[0].Fields[0].Len())
require.Equal(t, 1.5, *frames[0].Fields[1].At(3).(*float64))
@@ -447,236 +445,236 @@ func TestMSSQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using epoch (int64) as time column and value column (int64) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeInt64 as time, timeInt64 FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeInt64 as time, timeInt64 FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (int64 nullable) as time column and value column (int64 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeInt64Nullable as time, timeInt64Nullable FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeInt64Nullable as time, timeInt64Nullable FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (float64) as time column and value column (float64) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeFloat64 as time, timeFloat64 FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeFloat64 as time, timeFloat64 FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (float64 nullable) as time column and value column (float64 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeFloat64Nullable as time, timeFloat64Nullable FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeFloat64Nullable as time, timeFloat64Nullable FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (int32) as time column and value column (int32) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeInt32 as time, timeInt32 FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeInt32 as time, timeInt32 FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (int32 nullable) as time column and value column (int32 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeInt32Nullable as time, timeInt32Nullable FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeInt32Nullable as time, timeInt32Nullable FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (float32) as time column and value column (float32) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeFloat32 as time, timeFloat32 FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeFloat32 as time, timeFloat32 FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, tInitial, *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query using epoch (float32 nullable) as time column and value column (float32 nullable) should return metric with time in milliseconds", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT TOP 1 timeFloat32Nullable as time, timeFloat32Nullable FROM metric_values ORDER BY time`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT TOP 1 timeFloat32Nullable as time, timeFloat32Nullable FROM metric_values ORDER BY time",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, time.Unix(0, int64(float64(float32(tInitial.Unix()))*1e3)*int64(time.Millisecond)), *frames[0].Fields[0].At(0).(*time.Time))
})
t.Run("When doing a metric query grouping by time and select metric column should return correct series", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
- require.Equal(t, data.Labels{"metric": "Metric A - value one"}, frames[0].Fields[1].Labels)
- require.Equal(t, data.Labels{"metric": "Metric B - value one"}, frames[0].Fields[2].Labels)
+ require.Equal(t, string("Metric A - value one"), frames[0].Fields[1].Name)
+ require.Equal(t, string("Metric B - value one"), frames[0].Fields[2].Name)
})
t.Run("When doing a metric query grouping by time should return correct series", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
@@ -685,24 +683,24 @@ func TestMSSQL(t *testing.T) {
})
t.Run("When doing a metric query with metric column and multiple value columns", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeEpoch(time), measurement, valueOne, valueTwo FROM metric_values ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 5, len(frames[0].Fields))
@@ -717,26 +715,28 @@ func TestMSSQL(t *testing.T) {
})
t.Run("When doing a query with timeFrom,timeTo,unixEpochFrom,unixEpochTo macros", func(t *testing.T) {
- timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: fromStart}
- query := plugins.DataQuery{
- TimeRange: &timeRange,
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{JsonData: simplejson.New()},
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1`,
- "format": "time_series",
- }),
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-5 * time.Minute),
+ To: fromStart,
+ },
+ // here we may have to escape
+ JSON: []byte(`{
+ "rawSql": "SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, "SELECT time FROM metric_values WHERE time > '2018-03-15T12:55:00Z' OR time < '2018-03-15T12:55:00Z' OR 1 < 1521118500 OR 1521118800 > 1 ORDER BY 1", frames[0].Meta.ExecutedQueryString)
@@ -786,37 +786,40 @@ func TestMSSQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using stored procedure should return correct result", func(t *testing.T) {
- endpoint, err := NewExecutor(&models.DataSource{
- JsonData: simplejson.New(),
- SecureJsonData: securejsondata.SecureJsonData{},
- })
+ queryResultTransformer := mssqlQueryResultTransformer{
+ log: logger,
+ }
+ dsInfo := sqleng.DataSourceInfo{}
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "mssql",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
+ RowLimit: 1000000,
+ }
+ endpoint, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
require.NoError(t, err)
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{JsonData: simplejson.New()},
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `DECLARE
- @from int = $__unixEpochFrom(),
- @to int = $__unixEpochTo()
-
- EXEC dbo.sp_test_epoch @from, @to`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "DECLARE @from int = $__unixEpochFrom(), @to int = $__unixEpochTo() EXEC dbo.sp_test_epoch @from, @to",
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Unix(1521117000, 0),
+ To: time.Unix(1521122100, 0),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: "1521117000000",
- To: "1521122100000",
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 5, len(frames[0].Fields))
@@ -875,33 +878,28 @@ func TestMSSQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using stored procedure should return correct result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{JsonData: simplejson.New()},
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `DECLARE
- @from int = $__unixEpochFrom(),
- @to int = $__unixEpochTo()
-
- EXEC dbo.sp_test_epoch @from, @to`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "DECLARE @from int = $__unixEpochFrom(), @to int = $__unixEpochTo() EXEC dbo.sp_test_epoch @from, @to",
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Unix(1521117000, 0),
+ To: time.Unix(1521122100, 0),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: "1521117000000",
- To: "1521122100000",
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 5, len(frames[0].Fields))
@@ -963,53 +961,52 @@ func TestMSSQL(t *testing.T) {
}
t.Run("When doing an annotation query of deploy events should return expected result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{},
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT time_sec as time, description as [text], tags FROM [event] WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC",
- "format": "table",
- }),
+ "format": "table"
+ }`),
RefID: "Deploys",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-20 * time.Minute),
+ To: fromStart.Add(40 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["Deploys"]
- frames, err := queryResult.Dataframes.Decoded()
+ queryResult := resp.Responses["Deploys"]
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 3, frames[0].Fields[0].Len())
})
t.Run("When doing an annotation query of ticket events should return expected result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT time_sec as time, description as [text], tags FROM [event] WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC",
- "format": "table",
- }),
+ "format": "table"
+ }`),
RefID: "Tickets",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-20 * time.Minute),
+ To: fromStart.Add(40 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
- },
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["Tickets"]
- frames, err := queryResult.Dataframes.Decoded()
+ queryResult := resp.Responses["Tickets"]
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 3, frames[0].Fields[0].Len())
@@ -1018,29 +1015,22 @@ func TestMSSQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in datetime format", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
const dtFormat = "2006-01-02 15:04:05.999999999"
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT CAST('%s' AS DATETIME) as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Format(dtFormat))
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- CAST('%s' AS DATETIME) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Format(dtFormat)),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1, frames[0].Fields[0].Len())
@@ -1050,29 +1040,23 @@ func TestMSSQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch second format should return ms", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT %d as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix())
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- %d as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1, frames[0].Fields[0].Len())
@@ -1082,29 +1066,22 @@ func TestMSSQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch second format (int) should return ms", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT cast(%d as int) as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix())
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- cast(%d as int) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1, frames[0].Fields[0].Len())
@@ -1114,29 +1091,22 @@ func TestMSSQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch millisecond format should return ms", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT %d as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix()*1000)
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- %d as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()*1000),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1, frames[0].Fields[0].Len())
@@ -1145,28 +1115,24 @@ func TestMSSQL(t *testing.T) {
})
t.Run("When doing an annotation query with a time column holding a bigint null value should return nil", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT
- cast(null as bigint) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT cast(null as bigint) as time, 'message' as text, 'tag1,tag2' as tags",
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1, frames[0].Fields[0].Len())
@@ -1175,34 +1141,133 @@ func TestMSSQL(t *testing.T) {
})
t.Run("When doing an annotation query with a time column holding a datetime null value should return nil", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT
- cast(null as datetime) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT cast(null as datetime) as time, 'message' as text, 'tag1,tag2' as tags",
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := endpoint.DataQuery(context.Background(), nil, query)
+ resp, err := endpoint.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1, frames[0].Fields[0].Len())
// Should be in time.Time
require.Nil(t, frames[0].Fields[0].At(0))
})
+
+ t.Run("When doing an annotation query with a time and timeend column should return two fields of type time", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1631053772276 as time, 1631054012276 as timeend, '' as text, '' as tags",
+ "format": "table"
+ }`),
+ RefID: "A",
+ },
+ },
+ }
+
+ resp, err := endpoint.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+
+ frames := queryResult.Frames
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 4, len(frames[0].Fields))
+
+ require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[0].Type())
+ require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[1].Type())
+ })
+
+ t.Run("When row limit set to 1", func(t *testing.T) {
+ queryResultTransformer := mssqlQueryResultTransformer{
+ log: logger,
+ }
+ dsInfo := sqleng.DataSourceInfo{}
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "mssql",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"VARCHAR", "CHAR", "NVARCHAR", "NCHAR"},
+ RowLimit: 1,
+ }
+
+ handler, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMssqlMacroEngine(), logger)
+ require.NoError(t, err)
+
+ t.Run("When doing a table query that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1 as value UNION ALL select 2 as value",
+ "format": "table"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Now(),
+ To: time.Now(),
+ },
+ },
+ },
+ }
+
+ resp, err := handler.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+ frames := queryResult.Frames
+ require.NoError(t, err)
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 1, len(frames[0].Fields))
+ require.Equal(t, 1, frames[0].Rows())
+ require.Len(t, frames[0].Meta.Notices, 1)
+ require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
+ })
+
+ t.Run("When doing a time series that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1 as time, 1 as value UNION ALL select 2 as time, 2 as value",
+ "format": "time_series"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Now(),
+ To: time.Now(),
+ },
+ },
+ },
+ }
+
+ resp, err := handler.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+ frames := queryResult.Frames
+ require.NoError(t, err)
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 2, len(frames[0].Fields))
+ require.Equal(t, 1, frames[0].Rows())
+ require.Len(t, frames[0].Meta.Notices, 1)
+ require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
+ })
+ })
})
}
@@ -1231,47 +1296,47 @@ func TestTransformQueryError(t *testing.T) {
func TestGenerateConnectionString(t *testing.T) {
testCases := []struct {
desc string
- dataSource *models.DataSource
+ dataSource sqleng.DataSourceInfo
expConnStr string
}{
{
desc: "From URL w/ port",
- dataSource: &models.DataSource{
- Url: "localhost:1001",
+ dataSource: sqleng.DataSourceInfo{
+ URL: "localhost:1001",
Database: "database",
User: "user",
- JsonData: simplejson.NewFromAny(map[string]interface{}{}),
+ JsonData: sqleng.JsonData{},
},
expConnStr: "server=localhost;database=database;user id=user;password=;port=1001;",
},
// When no port is specified, the driver should be allowed to choose
{
desc: "From URL w/o port",
- dataSource: &models.DataSource{
- Url: "localhost",
+ dataSource: sqleng.DataSourceInfo{
+ URL: "localhost",
Database: "database",
User: "user",
- JsonData: simplejson.NewFromAny(map[string]interface{}{}),
+ JsonData: sqleng.JsonData{},
},
expConnStr: "server=localhost;database=database;user id=user;password=;",
},
// Port 0 should be equivalent to not specifying a port, i.e. let the driver choose
{
desc: "From URL w port 0",
- dataSource: &models.DataSource{
- Url: "localhost:0",
+ dataSource: sqleng.DataSourceInfo{
+ URL: "localhost:0",
Database: "database",
User: "user",
- JsonData: simplejson.NewFromAny(map[string]interface{}{}),
+ JsonData: sqleng.JsonData{},
},
expConnStr: "server=localhost;database=database;user id=user;password=;",
},
{
desc: "Defaults",
- dataSource: &models.DataSource{
+ dataSource: sqleng.DataSourceInfo{
Database: "database",
User: "user",
- JsonData: simplejson.NewFromAny(map[string]interface{}{}),
+ JsonData: sqleng.JsonData{},
},
expConnStr: "server=localhost;database=database;user id=user;password=;",
},
diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go
index 0d70815f2bd..634cb348a70 100644
--- a/pkg/tsdb/mysql/macros.go
+++ b/pkg/tsdb/mysql/macros.go
@@ -6,9 +6,9 @@ import (
"regexp"
"strings"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/gtime"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
@@ -26,7 +26,7 @@ func newMysqlMacroEngine(logger log.Logger) sqleng.SQLMacroEngine {
return &mySQLMacroEngine{SQLMacroEngineBase: sqleng.NewSQLMacroEngineBase(), logger: logger}
}
-func (m *mySQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, sql string) (string, error) {
+func (m *mySQLMacroEngine) Interpolate(query *backend.DataQuery, timeRange backend.TimeRange, sql string) (string, error) {
matches := restrictedRegExp.FindAllStringSubmatch(sql, 1)
if len(matches) > 0 {
m.logger.Error("show grants, session_user(), current_user(), system_user() or user() not allowed in query")
@@ -57,7 +57,7 @@ func (m *mySQLMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plu
return sql, nil
}
-func (m *mySQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query plugins.DataSubQuery, name string, args []string) (string, error) {
+func (m *mySQLMacroEngine) evaluateMacro(timeRange backend.TimeRange, query *backend.DataQuery, name string, args []string) (string, error) {
switch name {
case "__timeEpoch", "__time":
if len(args) == 0 {
@@ -69,11 +69,11 @@ func (m *mySQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", args[0], timeRange.GetFromAsSecondsEpoch(), timeRange.GetToAsSecondsEpoch()), nil
+ return fmt.Sprintf("%s BETWEEN FROM_UNIXTIME(%d) AND FROM_UNIXTIME(%d)", args[0], timeRange.From.UTC().Unix(), timeRange.To.UTC().Unix()), nil
case "__timeFrom":
- return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.GetFromAsSecondsEpoch()), nil
+ return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.From.UTC().Unix()), nil
case "__timeTo":
- return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.GetToAsSecondsEpoch()), nil
+ return fmt.Sprintf("FROM_UNIXTIME(%d)", timeRange.To.UTC().Unix()), nil
case "__timeGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval", name)
@@ -99,16 +99,16 @@ func (m *mySQLMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsSecondsEpoch(), args[0], timeRange.GetToAsSecondsEpoch()), nil
+ return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().Unix(), args[0], timeRange.To.UTC().Unix()), nil
case "__unixEpochNanoFilter":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsTimeUTC().UnixNano(), args[0], timeRange.GetToAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().UnixNano(), args[0], timeRange.To.UTC().UnixNano()), nil
case "__unixEpochNanoFrom":
- return fmt.Sprintf("%d", timeRange.GetFromAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%d", timeRange.From.UTC().UnixNano()), nil
case "__unixEpochNanoTo":
- return fmt.Sprintf("%d", timeRange.GetToAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%d", timeRange.To.UTC().UnixNano()), nil
case "__unixEpochGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name)
diff --git a/pkg/tsdb/mysql/macros_test.go b/pkg/tsdb/mysql/macros_test.go
index 275b35cfa78..c4546104cc6 100644
--- a/pkg/tsdb/mysql/macros_test.go
+++ b/pkg/tsdb/mysql/macros_test.go
@@ -2,14 +2,12 @@ package mysql
import (
"fmt"
- "strconv"
"sync"
"testing"
"time"
- "github.com/grafana/grafana/pkg/components/simplejson"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/plugins"
. "github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/require"
)
@@ -19,12 +17,12 @@ func TestMacroEngine(t *testing.T) {
engine := &mySQLMacroEngine{
logger: log.New("test"),
}
- query := plugins.DataSubQuery{}
+ query := &backend.DataQuery{}
Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", Now: to, To: "now"}
+ timeRange := backend.TimeRange{From: from, To: to}
Convey("interpolate __time function", func() {
sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)")
@@ -123,8 +121,10 @@ func TestMacroEngine(t *testing.T) {
Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -144,8 +144,10 @@ func TestMacroEngine(t *testing.T) {
Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
Convey("interpolate __timeFilter function", func() {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -185,7 +187,7 @@ func TestMacroEngine(t *testing.T) {
}
for _, tc := range tcs {
- _, err := engine.Interpolate(plugins.DataSubQuery{}, plugins.DataTimeRange{}, tc)
+ _, err := engine.Interpolate(&backend.DataQuery{}, backend.TimeRange{}, tc)
So(err.Error(), ShouldEqual, "invalid query - inspect Grafana server log for details")
}
})
@@ -194,27 +196,27 @@ func TestMacroEngine(t *testing.T) {
func TestMacroEngineConcurrency(t *testing.T) {
engine := newMysqlMacroEngine(log.New("test"))
- query1 := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query1 := backend.DataQuery{
+ JSON: []byte{},
}
- query2 := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query2 := backend.DataQuery{
+ JSON: []byte{},
}
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
+ timeRange := backend.TimeRange{From: from, To: to}
var wg sync.WaitGroup
wg.Add(2)
- go func(query plugins.DataSubQuery) {
+ go func(query backend.DataQuery) {
defer wg.Done()
- _, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+ _, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
}(query1)
- go func(query plugins.DataSubQuery) {
- _, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+ go func(query backend.DataQuery) {
+ _, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
defer wg.Done()
}(query2)
diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go
index bdb42fca232..531a97f6a9f 100644
--- a/pkg/tsdb/mysql/mysql.go
+++ b/pkg/tsdb/mysql/mysql.go
@@ -1,6 +1,8 @@
package mysql
import (
+ "context"
+ "encoding/json"
"errors"
"fmt"
"net/url"
@@ -10,15 +12,18 @@ import (
"time"
"github.com/VividCortex/mysqlerr"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
+ "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
"github.com/grafana/grafana/pkg/infra/httpclient"
+ "github.com/grafana/grafana/pkg/plugins/backendplugin"
+ "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
"github.com/grafana/grafana/pkg/setting"
"github.com/go-sql-driver/mysql"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
@@ -28,69 +33,127 @@ const (
dateTimeFormat2 = "2006-01-02T15:04:05Z"
)
+var logger = log.New("tsdb.mysql")
+
+type Service struct {
+ Cfg *setting.Cfg
+ im instancemgmt.InstanceManager
+}
+
func characterEscape(s string, escapeChar string) string {
return strings.ReplaceAll(s, escapeChar, url.QueryEscape(escapeChar))
}
-//nolint: staticcheck // plugins.DataPlugin deprecated
-func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSource) (plugins.DataPlugin, error) {
- //nolint: staticcheck // plugins.DataPlugin deprecated
- return func(datasource *models.DataSource) (plugins.DataPlugin, error) {
- logger := log.New("tsdb.mysql")
+func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager, httpClientProvider httpclient.Provider) (*Service, error) {
+ s := &Service{
+ im: datasource.NewInstanceManager(newInstanceSettings(cfg, httpClientProvider)),
+ }
+ factory := coreplugin.New(backend.ServeOpts{
+ QueryDataHandler: s,
+ })
+
+ if err := manager.Register("mysql", factory); err != nil {
+ logger.Error("Failed to register plugin", "error", err)
+ }
+ return s, nil
+}
+
+func newInstanceSettings(cfg *setting.Cfg, httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
+ return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+ jsonData := sqleng.JsonData{
+ MaxOpenConns: 0,
+ MaxIdleConns: 2,
+ ConnMaxLifetime: 14400,
+ }
+
+ err := json.Unmarshal(settings.JSONData, &jsonData)
+ if err != nil {
+ return nil, fmt.Errorf("error reading settings: %w", err)
+ }
+ dsInfo := sqleng.DataSourceInfo{
+ JsonData: jsonData,
+ URL: settings.URL,
+ User: settings.User,
+ Database: settings.Database,
+ ID: settings.ID,
+ Updated: settings.Updated,
+ UID: settings.UID,
+ DecryptedSecureJSONData: settings.DecryptedSecureJSONData,
+ }
protocol := "tcp"
- if strings.HasPrefix(datasource.Url, "/") {
+ if strings.HasPrefix(dsInfo.URL, "/") {
protocol = "unix"
}
cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
- characterEscape(datasource.User, ":"),
- datasource.DecryptedPassword(),
+ characterEscape(dsInfo.User, ":"),
+ dsInfo.DecryptedSecureJSONData["password"],
protocol,
- characterEscape(datasource.Url, ")"),
- characterEscape(datasource.Database, "?"),
+ characterEscape(dsInfo.URL, ")"),
+ characterEscape(dsInfo.Database, "?"),
)
- tlsConfig, err := datasource.GetTLSConfig(httpClientProvider)
+ opts, err := settings.HTTPClientOptions()
+ if err != nil {
+ return nil, err
+ }
+
+ tlsConfig, err := httpClientProvider.GetTLSConfig(opts)
if err != nil {
return nil, err
}
if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
- tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
+ tlsConfigString := fmt.Sprintf("ds%d", settings.ID)
if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil {
return nil, err
}
cnnstr += "&tls=" + tlsConfigString
}
- if datasource.JsonData != nil {
- timezone, hasTimezone := datasource.JsonData.CheckGet("timezone")
- if hasTimezone && timezone.MustString() != "" {
- cnnstr += fmt.Sprintf("&time_zone='%s'", url.QueryEscape(timezone.MustString()))
- }
+ if dsInfo.JsonData.Timezone != "" {
+ cnnstr += fmt.Sprintf("&time_zone='%s'", url.QueryEscape(dsInfo.JsonData.Timezone))
}
- if setting.Env == setting.Dev {
+ if cfg.Env == setting.Dev {
logger.Debug("getEngine", "connection", cnnstr)
}
config := sqleng.DataPluginConfiguration{
DriverName: "mysql",
ConnectionString: cnnstr,
- Datasource: datasource,
+ DSInfo: dsInfo,
TimeColumnNames: []string{"time", "time_sec"},
MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
+ RowLimit: cfg.DataProxyRowLimit,
}
rowTransformer := mysqlQueryResultTransformer{
log: logger,
}
- return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
+ return sqleng.NewQueryDataHandler(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
}
}
+func (s *Service) getDataSourceHandler(pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) {
+ i, err := s.im.Get(pluginCtx)
+ if err != nil {
+ return nil, err
+ }
+ instance := i.(*sqleng.DataSourceHandler)
+ return instance, nil
+}
+
+func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+ dsHandler, err := s.getDataSourceHandler(req.PluginContext)
+ if err != nil {
+ return nil, err
+ }
+ return dsHandler.QueryData(ctx, req)
+}
+
type mysqlQueryResultTransformer struct {
log log.Logger
}
diff --git a/pkg/tsdb/mysql/mysql_test.go b/pkg/tsdb/mysql/mysql_test.go
index 1881918fb88..8f021957a4a 100644
--- a/pkg/tsdb/mysql/mysql_test.go
+++ b/pkg/tsdb/mysql/mysql_test.go
@@ -11,12 +11,8 @@ import (
"testing"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
- "github.com/grafana/grafana/pkg/components/securejsondata"
- "github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/infra/httpclient"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
@@ -54,14 +50,33 @@ func TestMySQL(t *testing.T) {
return x, nil
}
- sqleng.Interpolate = func(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, sql string) (string, error) {
+ sqleng.Interpolate = func(query backend.DataQuery, timeRange backend.TimeRange, timeInterval string, sql string) (string, error) {
return sql, nil
}
- exe, err := New(httpclient.NewProvider())(&models.DataSource{
- JsonData: simplejson.New(),
- SecureJsonData: securejsondata.SecureJsonData{},
- })
+ dsInfo := sqleng.DataSourceInfo{
+ JsonData: sqleng.JsonData{
+ MaxOpenConns: 0,
+ MaxIdleConns: 2,
+ ConnMaxLifetime: 14400,
+ },
+ }
+
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "mysql",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ TimeColumnNames: []string{"time", "time_sec"},
+ MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
+ RowLimit: 1000000,
+ }
+
+ rowTransformer := mysqlQueryResultTransformer{
+ log: logger,
+ }
+
+ exe, err := sqleng.NewQueryDataHandler(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
+
require.NoError(t, err)
sess := x.NewSession()
@@ -125,23 +140,23 @@ func TestMySQL(t *testing.T) {
require.NoError(t, err)
t.Run("Query with Table format should map MySQL column types to Go types", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT * FROM mysql_types",
- "format": "table",
- }),
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Len(t, frames, 1)
@@ -218,24 +233,24 @@ func TestMySQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using timeGroup", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m') as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
// without fill this should result in 4 buckets
require.Equal(t, 4, frames[0].Fields[0].Len())
@@ -262,28 +277,28 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing a metric query using timeGroup with NULL fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', NULL) as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 7, frames[0].Fields[0].Len())
@@ -323,56 +338,55 @@ func TestMySQL(t *testing.T) {
})
t.Run("Should replace $__interval", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{JsonData: simplejson.New()},
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, $__interval) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(30 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(30*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, "SELECT UNIX_TIMESTAMP(time) DIV 60 * 60 AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", frames[0].Meta.ExecutedQueryString)
})
})
t.Run("When doing a metric query using timeGroup with value fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', 1.5) as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, data.TimeSeriesTimeFieldName, frames[0].Fields[0].Name)
require.Equal(t, 7, frames[0].Fields[0].Len())
@@ -380,28 +394,28 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing a metric query using timeGroup with previous fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', previous) as time_sec, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, float64(15.0), *frames[0].Fields[1].At(2).(*float64))
require.Equal(t, float64(15.0), *frames[0].Fields[1].At(3).(*float64))
@@ -487,24 +501,24 @@ func TestMySQL(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using time as time column should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time, valueOne FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT time, valueOne FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Len(t, frames, 1)
require.Equal(t, data.TimeSeriesTimeFieldName, frames[0].Fields[0].Name)
@@ -512,281 +526,281 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing a metric query using tinyint as value column should return metric with value in *float64", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time, valueThree FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT time, valueThree FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Len(t, frames, 1)
require.Equal(t, float64(6), *frames[0].Fields[1].At(0).(*float64))
})
t.Run("When doing a metric query using smallint as value column should return metric with value in *float64", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time, valueFour FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT time, valueFour FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Len(t, frames, 1)
require.Equal(t, float64(8), *frames[0].Fields[1].At(0).(*float64))
})
t.Run("When doing a metric query using time (nullable) as time column should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeNullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeNullable as time, valueOne FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int64) as time column and value column (int64) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeInt64 as time, timeInt64 FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeInt64 as time, timeInt64 FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int64 nullable) as time column and value column (int64 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeInt64Nullable as time, timeInt64Nullable FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeInt64Nullable as time, timeInt64Nullable FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float64) as time column and value column (float64) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeFloat64 as time, timeFloat64 FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeFloat64 as time, timeFloat64 FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float64 nullable) as time column and value column (float64 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeFloat64Nullable as time, timeFloat64Nullable FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeFloat64Nullable as time, timeFloat64Nullable FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int32) as time column and value column (int32) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeInt32 as time, timeInt32 FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeInt32 as time, timeInt32 FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int32 nullable) as time column and value column (int32 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeInt32Nullable as time, timeInt32Nullable FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeInt32Nullable as time, timeInt32Nullable FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float32) as time column and value column (float32) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeFloat32 as time, timeFloat32 FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeFloat32 as time, timeFloat32 FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
aTime := time.Unix(0, int64(float64(float32(tInitial.Unix()))*1e3)*int64(time.Millisecond))
require.True(t, aTime.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float32 nullable) as time column and value column (float32 nullable) should return metric with time in time.Time", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT timeFloat32Nullable as time, timeFloat32Nullable FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT timeFloat32Nullable as time, timeFloat32Nullable FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
aTime := time.Unix(0, int64(float64(float32(tInitial.Unix()))*1e3)*int64(time.Millisecond))
require.True(t, aTime.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query grouping by time and select metric column should return correct series", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT $__time(time), CONCAT(measurement, ' - value one') as metric, valueOne FROM metric_values ORDER BY 1,2`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__time(time), CONCAT(measurement, ' - value one') as metric, valueOne FROM metric_values ORDER BY 1,2",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 3)
require.Equal(t, "Metric A - value one", frames[0].Fields[1].Name)
@@ -794,24 +808,24 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing a metric query with metric column and multiple value columns", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values ORDER BY 1,2`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__time(time), measurement as metric, valueOne, valueTwo FROM metric_values ORDER BY 1,2",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 5)
@@ -826,24 +840,24 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing a metric query grouping by time should return correct series", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT $__time(time), valueOne, valueTwo FROM metric_values ORDER BY 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__time(time), valueOne, valueTwo FROM metric_values ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 3)
require.Equal(t, "valueOne", frames[0].Fields[1].Name)
@@ -853,25 +867,24 @@ func TestMySQL(t *testing.T) {
t.Run("When doing a query with timeFrom,timeTo,unixEpochFrom,unixEpochTo macros", func(t *testing.T) {
sqleng.Interpolate = origInterpolate
- query := plugins.DataQuery{
- TimeRange: &plugins.DataTimeRange{From: "5m", To: "now", Now: fromStart},
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{JsonData: simplejson.New()},
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeTo() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1`,
- "format": "time_series",
- }),
- RefID: "A",
+ JSON: []byte(`{
+ "rawSql": "SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeTo() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1",
+ "format": "time_series"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{From: fromStart.Add(-5 * time.Minute), To: fromStart},
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, "SELECT time FROM metric_values WHERE time > FROM_UNIXTIME(1521118500) OR time < FROM_UNIXTIME(1521118800) OR 1 < 1521118500 OR 1521118800 > 1 ORDER BY 1", frames[0].Meta.ExecutedQueryString)
})
@@ -912,53 +925,53 @@ func TestMySQL(t *testing.T) {
}
t.Run("When doing an annotation query of deploy events should return expected result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time_sec, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT time_sec, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC",
+ "format": "table"
+ }`),
RefID: "Deploys",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-20 * time.Minute),
+ To: fromStart.Add(40 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["Deploys"]
+ queryResult := resp.Responses["Deploys"]
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 3)
require.Equal(t, 3, frames[0].Fields[0].Len())
})
t.Run("When doing an annotation query of ticket events should return expected result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time_sec, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT time_sec, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC",
+ "format": "table"
+ }`),
RefID: "Tickets",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-20 * time.Minute),
+ To: fromStart.Add(40 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["Tickets"]
- frames, _ := queryResult.Dataframes.Decoded()
+ queryResult := resp.Responses["Tickets"]
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 3)
require.Equal(t, 3, frames[0].Fields[0].Len())
@@ -967,29 +980,22 @@ func TestMySQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in datetime format", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 0, time.UTC)
dtFormat := "2006-01-02 15:04:05.999999999"
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryJson := fmt.Sprintf("{\"rawSql\": \"SELECT CAST('%s' as datetime) as time_sec, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Format(dtFormat))
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- CAST('%s' as datetime) as time_sec,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Format(dtFormat)),
- "format": "table",
- }),
+ JSON: []byte(queryJson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 1, frames[0].Fields[0].Len())
//Should be in time.Time
@@ -998,29 +1004,22 @@ func TestMySQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch second format should return ms", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryJson := fmt.Sprintf("{\"rawSql\": \"SELECT %d as time_sec, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix())
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- %d as time_sec,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()),
- "format": "table",
- }),
+ JSON: []byte(queryJson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 1, frames[0].Fields[0].Len())
//Should be in time.Time
@@ -1029,29 +1028,22 @@ func TestMySQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch second format (signed integer) should return ms", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 0, time.Local)
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryJson := fmt.Sprintf("{\"rawSql\": \"SELECT CAST('%d' as signed integer) as time_sec, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix())
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- CAST('%d' as signed integer) as time_sec,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()),
- "format": "table",
- }),
+ JSON: []byte(queryJson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 1, frames[0].Fields[0].Len())
//Should be in time.Time
@@ -1060,29 +1052,23 @@ func TestMySQL(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch millisecond format should return ms", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
+ queryJson := fmt.Sprintf("{\"rawSql\": \"SELECT %d as time_sec, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix()*1000)
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- %d as time_sec,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()*1000),
- "format": "table",
- }),
+ JSON: []byte(queryJson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 1, frames[0].Fields[0].Len())
//Should be in time.Time
@@ -1090,28 +1076,24 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing an annotation query with a time column holding a unsigned integer null value should return nil", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT
- cast(null as unsigned integer) as time_sec,
- 'message' as text,
- 'tag1,tag2' as tags
- `,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT cast(null as unsigned integer) as time_sec, 'message' as text, 'tag1,tag2' as tags",
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 1, frames[0].Fields[0].Len())
@@ -1120,34 +1102,135 @@ func TestMySQL(t *testing.T) {
})
t.Run("When doing an annotation query with a time column holding a DATETIME null value should return nil", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT
- cast(null as DATETIME) as time_sec,
- 'message' as text,
- 'tag1,tag2' as tags
- `,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT cast(null as DATETIME) as time_sec, 'message' as text, 'tag1,tag2' as tags",
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 1, frames[0].Fields[0].Len())
//Should be in time.Time
require.Nil(t, frames[0].Fields[0].At(0))
})
+
+ t.Run("When doing an annotation query with a time and timeend column should return two fields of type time", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1631053772276 as time, 1631054012276 as timeend, '' as text, '' as tags",
+ "format": "table"
+ }`),
+ RefID: "A",
+ },
+ },
+ }
+
+ resp, err := exe.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+
+ frames := queryResult.Frames
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 4, len(frames[0].Fields))
+
+ require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[0].Type())
+ require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[1].Type())
+ })
+
+ t.Run("When row limit set to 1", func(t *testing.T) {
+ dsInfo := sqleng.DataSourceInfo{}
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "mysql",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ TimeColumnNames: []string{"time", "time_sec"},
+ MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
+ RowLimit: 1,
+ }
+
+ queryResultTransformer := mysqlQueryResultTransformer{
+ log: logger,
+ }
+
+ handler, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newMysqlMacroEngine(logger), logger)
+ require.NoError(t, err)
+
+ t.Run("When doing a table query that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1 as value UNION ALL select 2 as value",
+ "format": "table"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Now(),
+ To: time.Now(),
+ },
+ },
+ },
+ }
+
+ resp, err := handler.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+ frames := queryResult.Frames
+ require.NoError(t, err)
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 1, len(frames[0].Fields))
+ require.Equal(t, 1, frames[0].Rows())
+ require.Len(t, frames[0].Meta.Notices, 1)
+ require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
+ })
+
+ t.Run("When doing a time series that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1 as time, 1 as value UNION ALL select 2 as time, 2 as value",
+ "format": "time_series"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Now(),
+ To: time.Now(),
+ },
+ },
+ },
+ }
+
+ resp, err := handler.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+ frames := queryResult.Frames
+ require.NoError(t, err)
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 2, len(frames[0].Fields))
+ require.Equal(t, 1, frames[0].Rows())
+ require.Len(t, frames[0].Meta.Notices, 1)
+ require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
+ })
+ })
})
}
diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go
index c0ccc93f704..9fe7479e060 100644
--- a/pkg/tsdb/postgres/macros.go
+++ b/pkg/tsdb/postgres/macros.go
@@ -6,8 +6,8 @@ import (
"strings"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/components/gtime"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
@@ -26,8 +26,7 @@ func newPostgresMacroEngine(timescaledb bool) sqleng.SQLMacroEngine {
}
}
-func (m *postgresMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange,
- sql string) (string, error) {
+func (m *postgresMacroEngine) Interpolate(query *backend.DataQuery, timeRange backend.TimeRange, sql string) (string, error) {
// TODO: Handle error
rExp, _ := regexp.Compile(sExpr)
var macroError error
@@ -67,7 +66,7 @@ func (m *postgresMacroEngine) Interpolate(query plugins.DataSubQuery, timeRange
}
//nolint: gocyclo
-func (m *postgresMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, query plugins.DataSubQuery, name string, args []string) (string, error) {
+func (m *postgresMacroEngine) evaluateMacro(timeRange backend.TimeRange, query *backend.DataQuery, name string, args []string) (string, error) {
switch name {
case "__time":
if len(args) == 0 {
@@ -84,11 +83,11 @@ func (m *postgresMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, que
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], timeRange.GetFromAsTimeUTC().Format(time.RFC3339Nano), timeRange.GetToAsTimeUTC().Format(time.RFC3339Nano)), nil
+ return fmt.Sprintf("%s BETWEEN '%s' AND '%s'", args[0], timeRange.From.UTC().Format(time.RFC3339Nano), timeRange.To.UTC().Format(time.RFC3339Nano)), nil
case "__timeFrom":
- return fmt.Sprintf("'%s'", timeRange.GetFromAsTimeUTC().Format(time.RFC3339Nano)), nil
+ return fmt.Sprintf("'%s'", timeRange.From.UTC().Format(time.RFC3339Nano)), nil
case "__timeTo":
- return fmt.Sprintf("'%s'", timeRange.GetToAsTimeUTC().Format(time.RFC3339Nano)), nil
+ return fmt.Sprintf("'%s'", timeRange.To.UTC().Format(time.RFC3339Nano)), nil
case "__timeGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name)
@@ -123,16 +122,16 @@ func (m *postgresMacroEngine) evaluateMacro(timeRange plugins.DataTimeRange, que
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsSecondsEpoch(), args[0], timeRange.GetToAsSecondsEpoch()), nil
+ return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().Unix(), args[0], timeRange.To.UTC().Unix()), nil
case "__unixEpochNanoFilter":
if len(args) == 0 {
return "", fmt.Errorf("missing time column argument for macro %v", name)
}
- return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.GetFromAsTimeUTC().UnixNano(), args[0], timeRange.GetToAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], timeRange.From.UTC().UnixNano(), args[0], timeRange.To.UTC().UnixNano()), nil
case "__unixEpochNanoFrom":
- return fmt.Sprintf("%d", timeRange.GetFromAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%d", timeRange.From.UTC().UnixNano()), nil
case "__unixEpochNanoTo":
- return fmt.Sprintf("%d", timeRange.GetToAsTimeUTC().UnixNano()), nil
+ return fmt.Sprintf("%d", timeRange.To.UTC().UnixNano()), nil
case "__unixEpochGroup":
if len(args) < 2 {
return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name)
diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go
index 882dda43242..29a5207e0bb 100644
--- a/pkg/tsdb/postgres/macros_test.go
+++ b/pkg/tsdb/postgres/macros_test.go
@@ -2,13 +2,11 @@ package postgres
import (
"fmt"
- "strconv"
"sync"
"testing"
"time"
- "github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/plugins"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/stretchr/testify/require"
)
@@ -17,12 +15,12 @@ func TestMacroEngine(t *testing.T) {
engine := newPostgresMacroEngine(timescaledbEnabled)
timescaledbEnabled = true
engineTS := newPostgresMacroEngine(timescaledbEnabled)
- query := plugins.DataSubQuery{}
+ query := &backend.DataQuery{}
t.Run("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func(t *testing.T) {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
+ timeRange := backend.TimeRange{From: from, To: to}
t.Run("interpolate __time function", func(t *testing.T) {
sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)")
@@ -151,9 +149,10 @@ func TestMacroEngine(t *testing.T) {
t.Run("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func(t *testing.T) {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10),
- strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
t.Run("interpolate __timeFilter function", func(t *testing.T) {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -177,9 +176,10 @@ func TestMacroEngine(t *testing.T) {
t.Run("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func(t *testing.T) {
from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC)
to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10),
- strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
t.Run("interpolate __timeFilter function", func(t *testing.T) {
sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)")
@@ -203,9 +203,10 @@ func TestMacroEngine(t *testing.T) {
t.Run("Given a time range between 1960-02-01 07:00:00.5 and 1980-02-03 08:00:00.5", func(t *testing.T) {
from := time.Date(1960, 2, 1, 7, 0, 0, 500e6, time.UTC)
to := time.Date(1980, 2, 3, 8, 0, 0, 500e6, time.UTC)
- timeRange := plugins.NewDataTimeRange(
- strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10))
-
+ timeRange := backend.TimeRange{
+ From: from,
+ To: to,
+ }
require.Equal(t, "1960-02-01T07:00:00.5Z", from.Format(time.RFC3339Nano))
require.Equal(t, "1980-02-03T08:00:00.5Z", to.Format(time.RFC3339Nano))
@@ -219,27 +220,27 @@ func TestMacroEngine(t *testing.T) {
func TestMacroEngineConcurrency(t *testing.T) {
engine := newPostgresMacroEngine(false)
- query1 := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query1 := backend.DataQuery{
+ JSON: []byte{},
}
- query2 := plugins.DataSubQuery{
- Model: simplejson.New(),
+ query2 := backend.DataQuery{
+ JSON: []byte{},
}
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
+ timeRange := backend.TimeRange{From: from, To: to}
var wg sync.WaitGroup
wg.Add(2)
- go func(query plugins.DataSubQuery) {
+ go func(query backend.DataQuery) {
defer wg.Done()
- _, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+ _, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
}(query1)
- go func(query plugins.DataSubQuery) {
- _, err := engine.Interpolate(query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
+ go func(query backend.DataQuery) {
+ _, err := engine.Interpolate(&query, timeRange, "SELECT $__timeGroup(time_column,'5m')")
require.NoError(t, err)
defer wg.Done()
}(query2)
diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go
index 9fdd5a02b75..16af7c15ad8 100644
--- a/pkg/tsdb/postgres/postgres.go
+++ b/pkg/tsdb/postgres/postgres.go
@@ -1,72 +1,123 @@
package postgres
import (
+ "context"
+ "encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
+ "github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
+ "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
+ "github.com/grafana/grafana/pkg/plugins/backendplugin"
+ "github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
)
-func ProvideService(cfg *setting.Cfg) *PostgresService {
- logger := log.New("tsdb.postgres")
- return &PostgresService{
- Cfg: cfg,
- logger: logger,
+var logger = log.New("tsdb.postgres")
+
+func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager) (*Service, error) {
+ s := &Service{
tlsManager: newTLSManager(logger, cfg.DataPath),
}
+ s.im = datasource.NewInstanceManager(s.newInstanceSettings(cfg))
+ factory := coreplugin.New(backend.ServeOpts{
+ QueryDataHandler: s,
+ })
+
+ if err := manager.Register("postgres", factory); err != nil {
+ logger.Error("Failed to register plugin", "error", err)
+ }
+ return s, nil
}
-type PostgresService struct {
- Cfg *setting.Cfg
- logger log.Logger
+type Service struct {
tlsManager tlsSettingsProvider
+ im instancemgmt.InstanceManager
}
-//nolint: staticcheck // plugins.DataPlugin deprecated
-func (s *PostgresService) NewExecutor(datasource *models.DataSource) (plugins.DataPlugin, error) {
- s.logger.Debug("Creating Postgres query endpoint")
-
- cnnstr, err := s.generateConnectionString(datasource)
+func (s *Service) getDSInfo(pluginCtx backend.PluginContext) (*sqleng.DataSourceHandler, error) {
+ i, err := s.im.Get(pluginCtx)
if err != nil {
return nil, err
}
+ instance := i.(*sqleng.DataSourceHandler)
+ return instance, nil
+}
- if s.Cfg.Env == setting.Dev {
- s.logger.Debug("getEngine", "connection", cnnstr)
- }
-
- config := sqleng.DataPluginConfiguration{
- DriverName: "postgres",
- ConnectionString: cnnstr,
- Datasource: datasource,
- MetricColumnTypes: []string{"UNKNOWN", "TEXT", "VARCHAR", "CHAR"},
- }
-
- queryResultTransformer := postgresQueryResultTransformer{
- log: s.logger,
- }
-
- timescaledb := datasource.JsonData.Get("timescaledb").MustBool(false)
-
- plugin, err := sqleng.NewDataPlugin(config, &queryResultTransformer, newPostgresMacroEngine(timescaledb),
- s.logger)
+func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+ dsInfo, err := s.getDSInfo(req.PluginContext)
if err != nil {
- s.logger.Error("Failed connecting to Postgres", "err", err)
return nil, err
}
+ return dsInfo.QueryData(ctx, req)
+}
- s.logger.Debug("Successfully connected to Postgres")
- return plugin, nil
+func (s *Service) newInstanceSettings(cfg *setting.Cfg) datasource.InstanceFactoryFunc {
+ return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+ logger.Debug("Creating Postgres query endpoint")
+ jsonData := sqleng.JsonData{
+ MaxOpenConns: 0,
+ MaxIdleConns: 2,
+ ConnMaxLifetime: 14400,
+ Timescaledb: false,
+ ConfigurationMethod: "file-path",
+ }
+
+ err := json.Unmarshal(settings.JSONData, &jsonData)
+ if err != nil {
+ return nil, fmt.Errorf("error reading settings: %w", err)
+ }
+ dsInfo := sqleng.DataSourceInfo{
+ JsonData: jsonData,
+ URL: settings.URL,
+ User: settings.User,
+ Database: settings.Database,
+ ID: settings.ID,
+ Updated: settings.Updated,
+ UID: settings.UID,
+ DecryptedSecureJSONData: settings.DecryptedSecureJSONData,
+ }
+
+ cnnstr, err := s.generateConnectionString(dsInfo)
+ if err != nil {
+ return nil, err
+ }
+
+ if cfg.Env == setting.Dev {
+ logger.Debug("getEngine", "connection", cnnstr)
+ }
+
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "postgres",
+ ConnectionString: cnnstr,
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"UNKNOWN", "TEXT", "VARCHAR", "CHAR"},
+ RowLimit: cfg.DataProxyRowLimit,
+ }
+
+ queryResultTransformer := postgresQueryResultTransformer{
+ log: logger,
+ }
+
+ handler, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newPostgresMacroEngine(dsInfo.JsonData.Timescaledb),
+ logger)
+ if err != nil {
+ logger.Error("Failed connecting to Postgres", "err", err)
+ return nil, err
+ }
+
+ logger.Debug("Successfully connected to Postgres")
+ return handler, nil
+ }
}
// escape single quotes and backslashes in Postgres connection string parameters.
@@ -74,14 +125,14 @@ func escape(input string) string {
return strings.ReplaceAll(strings.ReplaceAll(input, `\`, `\\`), "'", `\'`)
}
-func (s *PostgresService) generateConnectionString(datasource *models.DataSource) (string, error) {
+func (s *Service) generateConnectionString(dsInfo sqleng.DataSourceInfo) (string, error) {
var host string
var port int
- if strings.HasPrefix(datasource.Url, "/") {
- host = datasource.Url
- s.logger.Debug("Generating connection string with Unix socket specifier", "socket", host)
+ if strings.HasPrefix(dsInfo.URL, "/") {
+ host = dsInfo.URL
+ logger.Debug("Generating connection string with Unix socket specifier", "socket", host)
} else {
- sp := strings.SplitN(datasource.Url, ":", 2)
+ sp := strings.SplitN(dsInfo.URL, ":", 2)
host = sp[0]
if len(sp) > 1 {
var err error
@@ -90,19 +141,19 @@ func (s *PostgresService) generateConnectionString(datasource *models.DataSource
return "", errutil.Wrapf(err, "invalid port in host specifier %q", sp[1])
}
- s.logger.Debug("Generating connection string with network host/port pair", "host", host, "port", port)
+ logger.Debug("Generating connection string with network host/port pair", "host", host, "port", port)
} else {
- s.logger.Debug("Generating connection string with network host", "host", host)
+ logger.Debug("Generating connection string with network host", "host", host)
}
}
connStr := fmt.Sprintf("user='%s' password='%s' host='%s' dbname='%s'",
- escape(datasource.User), escape(datasource.DecryptedPassword()), escape(host), escape(datasource.Database))
+ escape(dsInfo.User), escape(dsInfo.DecryptedSecureJSONData["password"]), escape(host), escape(dsInfo.Database))
if port > 0 {
connStr += fmt.Sprintf(" port=%d", port)
}
- tlsSettings, err := s.tlsManager.getTLSSettings(datasource)
+ tlsSettings, err := s.tlsManager.getTLSSettings(dsInfo)
if err != nil {
return "", err
}
@@ -111,19 +162,19 @@ func (s *PostgresService) generateConnectionString(datasource *models.DataSource
// Attach root certificate if provided
if tlsSettings.RootCertFile != "" {
- s.logger.Debug("Setting server root certificate", "tlsRootCert", tlsSettings.RootCertFile)
+ logger.Debug("Setting server root certificate", "tlsRootCert", tlsSettings.RootCertFile)
connStr += fmt.Sprintf(" sslrootcert='%s'", escape(tlsSettings.RootCertFile))
}
// Attach client certificate and key if both are provided
if tlsSettings.CertFile != "" && tlsSettings.CertKeyFile != "" {
- s.logger.Debug("Setting TLS/SSL client auth", "tlsCert", tlsSettings.CertFile, "tlsKey", tlsSettings.CertKeyFile)
+ logger.Debug("Setting TLS/SSL client auth", "tlsCert", tlsSettings.CertFile, "tlsKey", tlsSettings.CertKeyFile)
connStr += fmt.Sprintf(" sslcert='%s' sslkey='%s'", escape(tlsSettings.CertFile), escape(tlsSettings.CertKeyFile))
} else if tlsSettings.CertFile != "" || tlsSettings.CertKeyFile != "" {
return "", fmt.Errorf("TLS/SSL client certificate and key must both be specified")
}
- s.logger.Debug("Generated Postgres connection string successfully")
+ logger.Debug("Generated Postgres connection string successfully")
return connStr, nil
}
diff --git a/pkg/tsdb/postgres/postgres_test.go b/pkg/tsdb/postgres/postgres_test.go
index f581b0fc240..ae9e4baa2a9 100644
--- a/pkg/tsdb/postgres/postgres_test.go
+++ b/pkg/tsdb/postgres/postgres_test.go
@@ -11,12 +11,8 @@ import (
"testing"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
- "github.com/grafana/grafana/pkg/components/securejsondata"
- "github.com/grafana/grafana/pkg/components/simplejson"
- "github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
"github.com/grafana/grafana/pkg/setting"
@@ -115,18 +111,16 @@ func TestGenerateConnectionString(t *testing.T) {
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
- svc := PostgresService{
- Cfg: cfg,
- logger: log.New("tsdb.postgres"),
+ svc := Service{
tlsManager: &tlsTestManager{settings: tt.tlsSettings},
}
- ds := &models.DataSource{
- Url: tt.host,
- User: tt.user,
- Password: tt.password,
- Database: tt.database,
- Uid: tt.uid,
+ ds := sqleng.DataSourceInfo{
+ URL: tt.host,
+ User: tt.user,
+ DecryptedSecureJSONData: map[string]string{"password": tt.password},
+ Database: tt.database,
+ UID: tt.uid,
}
connStr, err := svc.generateConnectionString(ds)
@@ -170,22 +164,41 @@ func TestPostgres(t *testing.T) {
sqleng.NewXormEngine = func(d, c string) (*xorm.Engine, error) {
return x, nil
}
- sqleng.Interpolate = func(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, sql string) (string, error) {
+ sqleng.Interpolate = func(query backend.DataQuery, timeRange backend.TimeRange, timeInterval string, sql string) (string, error) {
return sql, nil
}
cfg := setting.NewCfg()
cfg.DataPath = t.TempDir()
- svc := PostgresService{
- Cfg: cfg,
- logger: log.New("tsdb.postgres"),
- tlsManager: &tlsTestManager{settings: tlsSettings{Mode: "disable"}},
+
+ jsonData := sqleng.JsonData{
+ MaxOpenConns: 0,
+ MaxIdleConns: 2,
+ ConnMaxLifetime: 14400,
+ Timescaledb: false,
+ ConfigurationMethod: "file-path",
}
- exe, err := svc.NewExecutor(&models.DataSource{
- JsonData: simplejson.New(),
- SecureJsonData: securejsondata.SecureJsonData{},
- })
+ dsInfo := sqleng.DataSourceInfo{
+ JsonData: jsonData,
+ DecryptedSecureJSONData: map[string]string{},
+ }
+
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "postgres",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"UNKNOWN", "TEXT", "VARCHAR", "CHAR"},
+ RowLimit: 1000000,
+ }
+
+ queryResultTransformer := postgresQueryResultTransformer{
+ log: logger,
+ }
+
+ exe, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newPostgresMacroEngine(dsInfo.JsonData.Timescaledb),
+ logger)
+
require.NoError(t, err)
sess := x.NewSession()
@@ -236,24 +249,23 @@ func TestPostgres(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a table query should map Postgres column types to Go types", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT * FROM postgres_types",
- "format": "table",
- }),
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
-
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 18)
@@ -326,24 +338,24 @@ func TestPostgres(t *testing.T) {
require.NoError(t, err)
t.Run("When doing a metric query using timeGroup", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t, 4, frames[0].Fields[0].Len())
@@ -377,27 +389,26 @@ func TestPostgres(t *testing.T) {
sqleng.Interpolate = mockInterpolate
})
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{},
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, $__interval) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(30 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(30*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
- frames, _ := queryResult.Dataframes.Decoded()
+ queryResult := resp.Responses["A"]
+ frames := queryResult.Frames
require.NoError(t, queryResult.Error)
require.Equal(t,
@@ -406,28 +417,28 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing a metric query using timeGroup with NULL fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 7, frames[0].Fields[0].Len())
@@ -460,28 +471,28 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing a metric query using timeGroup with value fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 1.5, *frames[0].Fields[1].At(3).(*float64))
})
@@ -505,28 +516,28 @@ func TestPostgres(t *testing.T) {
require.NoError(t, err)
t.Run("querying with time group with default value", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "WITH data AS (SELECT now()-'3m'::interval AS ts, 42 AS n) SELECT $__timeGroup(ts, '1m', 0), n FROM data",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: startTime,
+ To: startTime.Add(5 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", startTime.Unix()*1000),
- To: fmt.Sprintf("%v", startTime.Add(5*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, "Time", frames[0].Fields[0].Name)
require.Equal(t, "n", frames[0].Fields[1].Name)
@@ -540,28 +551,28 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing a metric query using timeGroup with previous fill enabled", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
+ JSON: []byte(`{
"rawSql": "SELECT $__timeGroup(time, '5m', previous), avg(value) as value FROM metric GROUP BY 1 ORDER BY 1",
- "format": "time_series",
- }),
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart,
+ To: fromStart.Add(34 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, float64(15.0), *frames[0].Fields[1].At(2).(*float64))
require.Equal(t, float64(15.0), *frames[0].Fields[1].At(3).(*float64))
@@ -638,168 +649,168 @@ func TestPostgres(t *testing.T) {
t.Run(
"When doing a metric query using epoch (int64) as time column and value column (int64) should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeInt64" as time, "timeInt64" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeInt64\" as time, \"timeInt64\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int64 nullable) as time column and value column (int64 nullable,) should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeInt64Nullable" as time, "timeInt64Nullable" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeInt64Nullable\" as time, \"timeInt64Nullable\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float64) as time column and value column (float64), should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeFloat64" as time, "timeFloat64" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeFloat64\" as time, \"timeFloat64\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float64 nullable) as time column and value column (float64 nullable), should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeFloat64Nullable" as time, "timeFloat64Nullable" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeFloat64Nullable\" as time, \"timeFloat64Nullable\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int32) as time column and value column (int32), should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeInt32" as time, "timeInt32" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeInt32\" as time, \"timeInt32\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (int32 nullable) as time column and value column (int32 nullable), should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeInt32Nullable" as time, "timeInt32Nullable" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeInt32Nullable\" as time, \"timeInt32Nullable\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.True(t, tInitial.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query using epoch (float32) as time column and value column (float32), should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeFloat32" as time, "timeFloat32" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeFloat32\" as time, \"timeFloat32\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
aTime := time.Unix(0, int64(float64(float32(tInitial.Unix()))*1e3)*int64(time.Millisecond))
require.True(t, aTime.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
@@ -807,48 +818,48 @@ func TestPostgres(t *testing.T) {
t.Run("When doing a metric query using epoch (float32 nullable) as time column and value column (float32 nullable), should return metric with time in time.Time",
func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "timeFloat32Nullable" as time, "timeFloat32Nullable" FROM metric_values ORDER BY time LIMIT 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"timeFloat32Nullable\" as time, \"timeFloat32Nullable\" FROM metric_values ORDER BY time LIMIT 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
aTime := time.Unix(0, int64(float64(float32(tInitial.Unix()))*1e3)*int64(time.Millisecond))
require.True(t, aTime.Equal(*frames[0].Fields[0].At(0).(*time.Time)))
})
t.Run("When doing a metric query grouping by time and select metric column should return correct series", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT $__timeEpoch(time), measurement || ' - value one' as metric, "valueOne" FROM metric_values ORDER BY 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__timeEpoch(time), measurement || ' - value one' as metric, \"valueOne\" FROM metric_values ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
require.Equal(t, "Metric A - value one", frames[0].Fields[1].Name)
@@ -856,24 +867,24 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing a metric query with metric column and multiple value columns", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT $__timeEpoch(time), measurement as metric, "valueOne", "valueTwo" FROM metric_values ORDER BY 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__timeEpoch(time), measurement as metric, \"valueOne\", \"valueTwo\" FROM metric_values ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, err := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.NoError(t, err)
require.Equal(t, 1, len(frames))
require.Equal(t, 5, len(frames[0].Fields))
@@ -888,24 +899,24 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing a metric query grouping by time should return correct series", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT $__timeEpoch(time), "valueOne", "valueTwo" FROM metric_values ORDER BY 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT $__timeEpoch(time), \"valueOne\", \"valueTwo\" FROM metric_values ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
require.Equal(t, "valueOne", frames[0].Fields[1].Name)
@@ -919,25 +930,27 @@ func TestPostgres(t *testing.T) {
})
sqleng.Interpolate = origInterpolate
- query := plugins.DataQuery{
- TimeRange: &plugins.DataTimeRange{From: "5m", To: "now", Now: fromStart},
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- DataSource: &models.DataSource{JsonData: simplejson.New()},
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1`,
- "format": "time_series",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT time FROM metric_values WHERE time > $__timeFrom() OR time < $__timeFrom() OR 1 < $__unixEpochFrom() OR $__unixEpochTo() > 1 ORDER BY 1",
+ "format": "time_series"
+ }`),
RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-5 * time.Minute),
+ To: fromStart,
+ },
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Equal(t,
"SELECT time FROM metric_values WHERE time > '2018-03-15T12:55:00Z' OR time < '2018-03-15T12:55:00Z' OR 1 < 1521118500 OR 1521118800 > 1 ORDER BY 1",
@@ -980,53 +993,54 @@ func TestPostgres(t *testing.T) {
}
t.Run("When doing an annotation query of deploy events should return expected result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"time_sec\" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC",
+ "format": "table"
+ }`),
RefID: "Deploys",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-20 * time.Minute),
+ To: fromStart.Add(40 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
- queryResult := resp.Results["Deploys"]
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- frames, _ := queryResult.Dataframes.Decoded()
+ queryResult := resp.Responses["Deploys"]
+
+ frames := queryResult.Frames
require.Len(t, frames, 1)
require.Len(t, frames[0].Fields, 3)
})
t.Run("When doing an annotation query of ticket events should return expected result", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT \"time_sec\" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC",
+ "format": "table"
+ }`),
RefID: "Tickets",
+ TimeRange: backend.TimeRange{
+ From: fromStart.Add(-20 * time.Minute),
+ To: fromStart.Add(40 * time.Minute),
+ },
},
},
- TimeRange: &plugins.DataTimeRange{
- From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000),
- To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000),
- },
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
- queryResult := resp.Results["Tickets"]
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- frames, _ := queryResult.Dataframes.Decoded()
+ queryResult := resp.Responses["Tickets"]
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
})
@@ -1035,27 +1049,21 @@ func TestPostgres(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
dtFormat := "2006-01-02 15:04:05.999999999"
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT CAST('%s' AS TIMESTAMP) as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\" }", dt.Format(dtFormat))
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- CAST('%s' AS TIMESTAMP) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Format(dtFormat)),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
@@ -1066,28 +1074,22 @@ func TestPostgres(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch second format should return time.Time", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT %d as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix())
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- %d as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
@@ -1097,29 +1099,22 @@ func TestPostgres(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch second format (t *testing.Tint) should return time.Time", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
-
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\": \"SELECT cast(%d as bigint) as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix())
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- cast(%d as bigint) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
@@ -1130,28 +1125,22 @@ func TestPostgres(t *testing.T) {
t.Run("When doing an annotation query with a time column in epoch millisecond format should return time.Time", func(t *testing.T) {
dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC)
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ queryjson := fmt.Sprintf("{\"rawSql\":\"SELECT %d as time, 'message' as text, 'tag1,tag2' as tags\", \"format\": \"table\"}", dt.Unix()*1000)
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": fmt.Sprintf(`SELECT
- %d as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `, dt.Unix()*1000),
- "format": "table",
- }),
+ JSON: []byte(queryjson),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
@@ -1160,28 +1149,24 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing an annotation query with a time column holding a bigint null value should return nil", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT
- cast(null as bigint) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT cast(null as bigint) as time, 'message' as text, 'tag1,tag2' as tags",
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
@@ -1190,34 +1175,134 @@ func TestPostgres(t *testing.T) {
})
t.Run("When doing an annotation query with a time column holding a timestamp null value should return nil", func(t *testing.T) {
- query := plugins.DataQuery{
- Queries: []plugins.DataSubQuery{
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
{
- Model: simplejson.NewFromAny(map[string]interface{}{
- "rawSql": `SELECT
- cast(null as timestamp) as time,
- 'message' as text,
- 'tag1,tag2' as tags
- `,
- "format": "table",
- }),
+ JSON: []byte(`{
+ "rawSql": "SELECT cast(null as timestamp) as time, 'message' as text, 'tag1,tag2' as tags",
+ "format": "table"
+ }`),
RefID: "A",
},
},
}
- resp, err := exe.DataQuery(context.Background(), nil, query)
+ resp, err := exe.QueryData(context.Background(), query)
require.NoError(t, err)
- queryResult := resp.Results["A"]
+ queryResult := resp.Responses["A"]
require.NoError(t, queryResult.Error)
- frames, _ := queryResult.Dataframes.Decoded()
+ frames := queryResult.Frames
require.Equal(t, 1, len(frames))
require.Equal(t, 3, len(frames[0].Fields))
// Should be in time.Time
assert.Nil(t, frames[0].Fields[0].At(0))
})
+
+ t.Run("When doing an annotation query with a time and timeend column should return two fields of type time", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1631053772276 as time, 1631054012276 as timeend, '' as text, '' as tags",
+ "format": "table"
+ }`),
+ RefID: "A",
+ },
+ },
+ }
+
+ resp, err := exe.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+
+ frames := queryResult.Frames
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 4, len(frames[0].Fields))
+
+ require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[0].Type())
+ require.Equal(t, data.FieldTypeNullableTime, frames[0].Fields[1].Type())
+ })
+
+ t.Run("When row limit set to 1", func(t *testing.T) {
+ dsInfo := sqleng.DataSourceInfo{}
+ config := sqleng.DataPluginConfiguration{
+ DriverName: "postgres",
+ ConnectionString: "",
+ DSInfo: dsInfo,
+ MetricColumnTypes: []string{"UNKNOWN", "TEXT", "VARCHAR", "CHAR"},
+ RowLimit: 1,
+ }
+
+ queryResultTransformer := postgresQueryResultTransformer{
+ log: logger,
+ }
+
+ handler, err := sqleng.NewQueryDataHandler(config, &queryResultTransformer, newPostgresMacroEngine(false), logger)
+ require.NoError(t, err)
+
+ t.Run("When doing a table query that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1 as value UNION ALL select 2 as value",
+ "format": "table"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Now(),
+ To: time.Now(),
+ },
+ },
+ },
+ }
+
+ resp, err := handler.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+ frames := queryResult.Frames
+ require.NoError(t, err)
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 1, len(frames[0].Fields))
+ require.Equal(t, 1, frames[0].Rows())
+ require.Len(t, frames[0].Meta.Notices, 1)
+ require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
+ })
+
+ t.Run("When doing a time series query that returns 2 rows should limit the result to 1 row", func(t *testing.T) {
+ query := &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(`{
+ "rawSql": "SELECT 1 as time, 1 as value UNION ALL select 2 as time, 2 as value",
+ "format": "time_series"
+ }`),
+ RefID: "A",
+ TimeRange: backend.TimeRange{
+ From: time.Now(),
+ To: time.Now(),
+ },
+ },
+ },
+ }
+
+ resp, err := handler.QueryData(context.Background(), query)
+ require.NoError(t, err)
+ queryResult := resp.Responses["A"]
+ require.NoError(t, queryResult.Error)
+ frames := queryResult.Frames
+ require.NoError(t, err)
+ require.Equal(t, 1, len(frames))
+ require.Equal(t, 2, len(frames[0].Fields))
+ require.Equal(t, 1, frames[0].Rows())
+ require.Len(t, frames[0].Meta.Notices, 1)
+ require.Equal(t, data.NoticeSeverityWarning, frames[0].Meta.Notices[0].Severity)
+ })
+ })
})
}
@@ -1252,6 +1337,6 @@ type tlsTestManager struct {
settings tlsSettings
}
-func (m *tlsTestManager) getTLSSettings(datasource *models.DataSource) (tlsSettings, error) {
+func (m *tlsTestManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
return m.settings, nil
}
diff --git a/pkg/tsdb/postgres/tlsmanager.go b/pkg/tsdb/postgres/tlsmanager.go
index eadb67bf491..08e79a8b155 100644
--- a/pkg/tsdb/postgres/tlsmanager.go
+++ b/pkg/tsdb/postgres/tlsmanager.go
@@ -8,17 +8,26 @@ import (
"strconv"
"strings"
"sync"
+ "time"
"github.com/grafana/grafana/pkg/infra/fs"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/tsdb/sqleng"
)
var validateCertFunc = validateCertFilePaths
var writeCertFileFunc = writeCertFile
+type certFileType int
+
+const (
+ rootCert = iota
+ clientCert
+ clientKey
+)
+
type tlsSettingsProvider interface {
- getTLSSettings(datasource *models.DataSource) (tlsSettings, error)
+ getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error)
}
type datasourceCacheManager struct {
@@ -48,46 +57,37 @@ type tlsSettings struct {
CertKeyFile string
}
-func (m *tlsManager) getTLSSettings(datasource *models.DataSource) (tlsSettings, error) {
- tlsMode := strings.TrimSpace(strings.ToLower(datasource.JsonData.Get("sslmode").MustString("verify-full")))
- isTLSDisabled := tlsMode == "disable"
+func (m *tlsManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
+ tlsconfig := tlsSettings{
+ Mode: dsInfo.JsonData.Mode,
+ }
- settings := tlsSettings{}
- settings.Mode = tlsMode
+ isTLSDisabled := (tlsconfig.Mode == "disable")
if isTLSDisabled {
m.logger.Debug("Postgres TLS/SSL is disabled")
- return settings, nil
+ return tlsconfig, nil
}
- m.logger.Debug("Postgres TLS/SSL is enabled", "tlsMode", tlsMode)
+ m.logger.Debug("Postgres TLS/SSL is enabled", "tlsMode", tlsconfig.Mode)
- settings.ConfigurationMethod = strings.TrimSpace(
- strings.ToLower(datasource.JsonData.Get("tlsConfigurationMethod").MustString("file-path")))
+ tlsconfig.ConfigurationMethod = dsInfo.JsonData.ConfigurationMethod
+ tlsconfig.RootCertFile = dsInfo.JsonData.RootCertFile
+ tlsconfig.CertFile = dsInfo.JsonData.CertFile
+ tlsconfig.CertKeyFile = dsInfo.JsonData.CertKeyFile
- if settings.ConfigurationMethod == "file-content" {
- if err := m.writeCertFiles(datasource, &settings); err != nil {
- return settings, err
+ if tlsconfig.ConfigurationMethod == "file-content" {
+ if err := m.writeCertFiles(dsInfo, &tlsconfig); err != nil {
+ return tlsconfig, err
}
} else {
- settings.RootCertFile = datasource.JsonData.Get("sslRootCertFile").MustString("")
- settings.CertFile = datasource.JsonData.Get("sslCertFile").MustString("")
- settings.CertKeyFile = datasource.JsonData.Get("sslKeyFile").MustString("")
- if err := validateCertFunc(settings.RootCertFile, settings.CertFile, settings.CertKeyFile); err != nil {
- return settings, err
+ if err := validateCertFunc(tlsconfig.RootCertFile, tlsconfig.CertFile, tlsconfig.CertKeyFile); err != nil {
+ return tlsconfig, err
}
}
- return settings, nil
+ return tlsconfig, nil
}
-type certFileType int
-
-const (
- rootCert = iota
- clientCert
- clientKey
-)
-
func (t certFileType) String() string {
switch t {
case rootCert:
@@ -118,8 +118,7 @@ func getFileName(dataDir string, fileType certFileType) string {
}
// writeCertFile writes a certificate file.
-func writeCertFile(
- ds *models.DataSource, logger log.Logger, fileContent string, generatedFilePath string) error {
+func writeCertFile(logger log.Logger, fileContent string, generatedFilePath string) error {
fileContent = strings.TrimSpace(fileContent)
if fileContent != "" {
logger.Debug("Writing cert file", "path", generatedFilePath)
@@ -146,30 +145,28 @@ func writeCertFile(
return nil
}
-func (m *tlsManager) writeCertFiles(ds *models.DataSource, settings *tlsSettings) error {
+func (m *tlsManager) writeCertFiles(dsInfo sqleng.DataSourceInfo, tlsconfig *tlsSettings) error {
m.logger.Debug("Writing TLS certificate files to disk")
- decrypted := ds.DecryptedValues()
- tlsRootCert := decrypted["tlsCACert"]
- tlsClientCert := decrypted["tlsClientCert"]
- tlsClientKey := decrypted["tlsClientKey"]
-
+ tlsRootCert := dsInfo.DecryptedSecureJSONData["tlsCACert"]
+ tlsClientCert := dsInfo.DecryptedSecureJSONData["tlsClientCert"]
+ tlsClientKey := dsInfo.DecryptedSecureJSONData["tlsClientKey"]
if tlsRootCert == "" && tlsClientCert == "" && tlsClientKey == "" {
m.logger.Debug("No TLS/SSL certificates provided")
}
// Calculate all files path
- workDir := filepath.Join(m.dataPath, "tls", ds.Uid+"generatedTLSCerts")
- settings.RootCertFile = getFileName(workDir, rootCert)
- settings.CertFile = getFileName(workDir, clientCert)
- settings.CertKeyFile = getFileName(workDir, clientKey)
+ workDir := filepath.Join(m.dataPath, "tls", dsInfo.UID+"generatedTLSCerts")
+ tlsconfig.RootCertFile = getFileName(workDir, rootCert)
+ tlsconfig.CertFile = getFileName(workDir, clientCert)
+ tlsconfig.CertKeyFile = getFileName(workDir, clientKey)
// Find datasource in the cache, if found, skip writing files
- cacheKey := strconv.Itoa(int(ds.Id))
+ cacheKey := strconv.Itoa(int(dsInfo.ID))
m.dsCacheInstance.locker.RLock(cacheKey)
item, ok := m.dsCacheInstance.cache.Load(cacheKey)
m.dsCacheInstance.locker.RUnlock(cacheKey)
if ok {
- if item.(int) == ds.Version {
+ if !item.(time.Time).Before(dsInfo.Updated) {
return nil
}
}
@@ -179,7 +176,7 @@ func (m *tlsManager) writeCertFiles(ds *models.DataSource, settings *tlsSettings
item, ok = m.dsCacheInstance.cache.Load(cacheKey)
if ok {
- if item.(int) == ds.Version {
+ if !item.(time.Time).Before(dsInfo.Updated) {
return nil
}
}
@@ -195,18 +192,18 @@ func (m *tlsManager) writeCertFiles(ds *models.DataSource, settings *tlsSettings
}
}
- if err = writeCertFileFunc(ds, m.logger, tlsRootCert, settings.RootCertFile); err != nil {
+ if err = writeCertFileFunc(m.logger, tlsRootCert, tlsconfig.RootCertFile); err != nil {
return err
}
- if err = writeCertFileFunc(ds, m.logger, tlsClientCert, settings.CertFile); err != nil {
+ if err = writeCertFileFunc(m.logger, tlsClientCert, tlsconfig.CertFile); err != nil {
return err
}
- if err = writeCertFileFunc(ds, m.logger, tlsClientKey, settings.CertKeyFile); err != nil {
+ if err = writeCertFileFunc(m.logger, tlsClientKey, tlsconfig.CertKeyFile); err != nil {
return err
}
// Update datasource cache
- m.dsCacheInstance.cache.Store(cacheKey, ds.Version)
+ m.dsCacheInstance.cache.Store(cacheKey, dsInfo.Updated)
return nil
}
diff --git a/pkg/tsdb/postgres/tlsmanager_test.go b/pkg/tsdb/postgres/tlsmanager_test.go
index d08b85bfe77..dd3fd3f6ed4 100644
--- a/pkg/tsdb/postgres/tlsmanager_test.go
+++ b/pkg/tsdb/postgres/tlsmanager_test.go
@@ -7,12 +7,11 @@ import (
"strings"
"sync"
"testing"
+ "time"
- "github.com/grafana/grafana/pkg/components/securejsondata"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
+ "github.com/grafana/grafana/pkg/tsdb/sqleng"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -30,16 +29,17 @@ func TestDataSourceCacheManager(t *testing.T) {
dsCacheInstance: datasourceCacheManager{locker: newLocker()},
dataPath: cfg.DataPath,
}
-
- jsonData := simplejson.NewFromAny(map[string]interface{}{
- "sslmode": "verify-full",
- "tlsConfigurationMethod": "file-content",
- })
- secureJSONData := securejsondata.GetEncryptedJsonData(map[string]string{
+ jsonData := sqleng.JsonData{
+ Mode: "verify-full",
+ ConfigurationMethod: "file-content",
+ }
+ secureJSONData := map[string]string{
"tlsClientCert": "I am client certification",
"tlsClientKey": "I am client key",
"tlsCACert": "I am CA certification",
- })
+ }
+
+ updateTime := time.Now().Add(-5 * time.Minute)
mockValidateCertFilePaths()
t.Cleanup(resetValidateCertFilePaths)
@@ -49,13 +49,13 @@ func TestDataSourceCacheManager(t *testing.T) {
wg.Add(10)
for id := int64(1); id <= 10; id++ {
go func(id int64) {
- ds := &models.DataSource{
- Id: id,
- Version: 1,
- Database: "database",
- JsonData: jsonData,
- SecureJsonData: secureJSONData,
- Uid: "testData",
+ ds := sqleng.DataSourceInfo{
+ ID: id,
+ Updated: updateTime,
+ Database: "database",
+ JsonData: jsonData,
+ DecryptedSecureJSONData: secureJSONData,
+ UID: "testData",
}
s := tlsSettings{}
err := mng.writeCertFiles(ds, &s)
@@ -67,9 +67,9 @@ func TestDataSourceCacheManager(t *testing.T) {
t.Run("check cache creation is succeed", func(t *testing.T) {
for id := int64(1); id <= 10; id++ {
- version, ok := mng.dsCacheInstance.cache.Load(strconv.Itoa(int(id)))
+ updated, ok := mng.dsCacheInstance.cache.Load(strconv.Itoa(int(id)))
require.True(t, ok)
- require.Equal(t, int(1), version)
+ require.Equal(t, updateTime, updated)
}
})
})
@@ -82,13 +82,13 @@ func TestDataSourceCacheManager(t *testing.T) {
wg1.Add(5)
for id := int64(1); id <= 5; id++ {
go func(id int64) {
- ds := &models.DataSource{
- Id: 1,
- Version: 2,
- Database: "database",
- JsonData: jsonData,
- SecureJsonData: secureJSONData,
- Uid: "testData",
+ ds := sqleng.DataSourceInfo{
+ ID: 1,
+ Updated: updateTime,
+ Database: "database",
+ JsonData: jsonData,
+ DecryptedSecureJSONData: secureJSONData,
+ UID: "testData",
}
s := tlsSettings{}
err := mng.writeCertFiles(ds, &s)
@@ -97,25 +97,25 @@ func TestDataSourceCacheManager(t *testing.T) {
}(id)
}
wg1.Wait()
- assert.Equal(t, writeCertFileCallNum, 3)
+ assert.Equal(t, writeCertFileCallNum, 0)
})
t.Run("cache is updated with the last datasource version", func(t *testing.T) {
- dsV2 := &models.DataSource{
- Id: 1,
- Version: 2,
- Database: "database",
- JsonData: jsonData,
- SecureJsonData: secureJSONData,
- Uid: "testData",
+ dsV2 := sqleng.DataSourceInfo{
+ ID: 1,
+ Updated: updateTime.Add(time.Minute),
+ Database: "database",
+ JsonData: jsonData,
+ DecryptedSecureJSONData: secureJSONData,
+ UID: "testData",
}
- dsV3 := &models.DataSource{
- Id: 1,
- Version: 3,
- Database: "database",
- JsonData: jsonData,
- SecureJsonData: secureJSONData,
- Uid: "testData",
+ dsV3 := sqleng.DataSourceInfo{
+ ID: 1,
+ Updated: updateTime.Add(2 * time.Minute),
+ Database: "database",
+ JsonData: jsonData,
+ DecryptedSecureJSONData: secureJSONData,
+ UID: "testData",
}
s := tlsSettings{}
err := mng.writeCertFiles(dsV2, &s)
@@ -124,7 +124,7 @@ func TestDataSourceCacheManager(t *testing.T) {
require.NoError(t, err)
version, ok := mng.dsCacheInstance.cache.Load("1")
require.True(t, ok)
- require.Equal(t, int(3), version)
+ require.Equal(t, updateTime.Add(2*time.Minute), version)
})
})
}
@@ -173,36 +173,39 @@ func TestGetTLSSettings(t *testing.T) {
mockValidateCertFilePaths()
t.Cleanup(resetValidateCertFilePaths)
+
+ updatedTime := time.Now()
+
testCases := []struct {
desc string
expErr string
- jsonData map[string]interface{}
+ jsonData sqleng.JsonData
secureJSONData map[string]string
uid string
tlsSettings tlsSettings
- version int
+ updated time.Time
}{
{
desc: "Custom TLS authentication disabled",
- version: 1,
- jsonData: map[string]interface{}{
- "sslmode": "disable",
- "sslRootCertFile": "i/am/coding/ca.crt",
- "sslCertFile": "i/am/coding/client.crt",
- "sslKeyFile": "i/am/coding/client.key",
- "tlsConfigurationMethod": "file-path",
+ updated: updatedTime,
+ jsonData: sqleng.JsonData{
+ Mode: "disable",
+ RootCertFile: "i/am/coding/ca.crt",
+ CertFile: "i/am/coding/client.crt",
+ CertKeyFile: "i/am/coding/client.key",
+ ConfigurationMethod: "file-path",
},
tlsSettings: tlsSettings{Mode: "disable"},
},
{
desc: "Custom TLS authentication with file path",
- version: 2,
- jsonData: map[string]interface{}{
- "sslmode": "verify-full",
- "sslRootCertFile": "i/am/coding/ca.crt",
- "sslCertFile": "i/am/coding/client.crt",
- "sslKeyFile": "i/am/coding/client.key",
- "tlsConfigurationMethod": "file-path",
+ updated: updatedTime.Add(time.Minute),
+ jsonData: sqleng.JsonData{
+ Mode: "verify-full",
+ ConfigurationMethod: "file-path",
+ RootCertFile: "i/am/coding/ca.crt",
+ CertFile: "i/am/coding/client.crt",
+ CertKeyFile: "i/am/coding/client.key",
},
tlsSettings: tlsSettings{
Mode: "verify-full",
@@ -214,11 +217,11 @@ func TestGetTLSSettings(t *testing.T) {
},
{
desc: "Custom TLS mode verify-full with certificate files content",
- version: 3,
+ updated: updatedTime.Add(2 * time.Minute),
uid: "xxx",
- jsonData: map[string]interface{}{
- "sslmode": "verify-full",
- "tlsConfigurationMethod": "file-content",
+ jsonData: sqleng.JsonData{
+ Mode: "verify-full",
+ ConfigurationMethod: "file-content",
},
secureJSONData: map[string]string{
"tlsCACert": "I am CA certification",
@@ -244,12 +247,11 @@ func TestGetTLSSettings(t *testing.T) {
dataPath: cfg.DataPath,
}
- jsonData := simplejson.NewFromAny(tt.jsonData)
- ds := &models.DataSource{
- JsonData: jsonData,
- SecureJsonData: securejsondata.GetEncryptedJsonData(tt.secureJSONData),
- Uid: tt.uid,
- Version: tt.version,
+ ds := sqleng.DataSourceInfo{
+ JsonData: tt.jsonData,
+ DecryptedSecureJSONData: tt.secureJSONData,
+ UID: tt.uid,
+ Updated: tt.updated,
}
settings, err = mng.getTLSSettings(ds)
@@ -278,7 +280,7 @@ func resetValidateCertFilePaths() {
func mockWriteCertFile() {
writeCertFileCallNum = 0
- writeCertFileFunc = func(ds *models.DataSource, logger log.Logger, fileContent string, generatedFilePath string) error {
+ writeCertFileFunc = func(logger log.Logger, fileContent string, generatedFilePath string) error {
writeCertFileCallNum++
return nil
}
diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go
index ad9163a88fc..786c0a81631 100644
--- a/pkg/tsdb/prometheus/prometheus.go
+++ b/pkg/tsdb/prometheus/prometheus.go
@@ -5,8 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
+ "math"
"net/http"
"regexp"
+ "strconv"
"strings"
"time"
@@ -15,12 +17,11 @@ import (
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/plugins/backendplugin/coreplugin"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/api"
apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
@@ -41,9 +42,20 @@ type DatasourceInfo struct {
TimeInterval string
}
+type QueryModel struct {
+ Expr string `json:"expr"`
+ LegendFormat string `json:"legendFormat"`
+ Interval string `json:"interval"`
+ IntervalMS int64 `json:"intervalMS"`
+ StepMode string `json:"stepMode"`
+ RangeQuery bool `json:"range"`
+ InstantQuery bool `json:"instant"`
+ IntervalFactor int64 `json:"intervalFactor"`
+}
+
type Service struct {
httpClientProvider httpclient.Provider
- intervalCalculator tsdb.Calculator
+ intervalCalculator intervalv2.Calculator
im instancemgmt.InstanceManager
}
@@ -53,7 +65,7 @@ func ProvideService(httpClientProvider httpclient.Provider, backendPluginManager
s := &Service{
httpClientProvider: httpClientProvider,
- intervalCalculator: tsdb.NewCalculator(),
+ intervalCalculator: intervalv2.NewCalculator(),
im: im,
}
@@ -130,7 +142,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
Responses: backend.Responses{},
}
- queries, err := s.parseQuery(req.Queries, dsInfo)
+ queries, err := s.parseQuery(req, dsInfo)
if err != nil {
return &result, err
}
@@ -225,64 +237,59 @@ func formatLegend(metric model.Metric, query *PrometheusQuery) string {
return string(result)
}
-func (s *Service) parseQuery(queries []backend.DataQuery, dsInfo *DatasourceInfo) (
- []*PrometheusQuery, error) {
- var intervalMode string
- var adjustedInterval time.Duration
-
+func (s *Service) parseQuery(queryContext *backend.QueryDataRequest, dsInfo *DatasourceInfo) ([]*PrometheusQuery, error) {
qs := []*PrometheusQuery{}
- for _, queryModel := range queries {
- jsonModel, err := simplejson.NewJson(queryModel.JSON)
- if err != nil {
- return nil, err
- }
- expr, err := jsonModel.Get("expr").String()
+ for _, query := range queryContext.Queries {
+ model := &QueryModel{}
+ err := json.Unmarshal(query.JSON, model)
if err != nil {
return nil, err
}
- format := jsonModel.Get("legendFormat").MustString("")
-
- start := queryModel.TimeRange.From
- end := queryModel.TimeRange.To
- queryInterval := jsonModel.Get("interval").MustString("")
-
- foundInterval, err := tsdb.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, 0, 15*time.Second)
- hasQueryInterval := queryInterval != ""
- // Only use stepMode if we have interval in query, otherwise use "min"
- if hasQueryInterval {
- intervalMode = jsonModel.Get("stepMode").MustString("min")
- } else {
- intervalMode = "min"
+ //Calculate interval
+ queryInterval := model.Interval
+ //If we are using variable or interval/step, we will replace it with calculated interval
+ if queryInterval == "$__interval" || queryInterval == "$__interval_ms" {
+ queryInterval = ""
}
-
- // Calculate interval value from query or data source settings or use default value
+ minInterval, err := intervalv2.GetIntervalFrom(dsInfo.TimeInterval, queryInterval, model.IntervalMS, 15*time.Second)
if err != nil {
return nil, err
}
- calculatedInterval, err := s.intervalCalculator.Calculate(queries[0].TimeRange, foundInterval, tsdb.IntervalMode(intervalMode))
- if err != nil {
- return nil, err
- }
- safeInterval := s.intervalCalculator.CalculateSafeInterval(queries[0].TimeRange, int64(safeRes))
+ calculatedInterval := s.intervalCalculator.Calculate(query.TimeRange, minInterval, query.MaxDataPoints)
+ safeInterval := s.intervalCalculator.CalculateSafeInterval(query.TimeRange, int64(safeRes))
+ adjustedInterval := safeInterval.Value
if calculatedInterval.Value > safeInterval.Value {
adjustedInterval = calculatedInterval.Value
- } else {
- adjustedInterval = safeInterval.Value
}
- intervalFactor := jsonModel.Get("intervalFactor").MustInt64(1)
- step := time.Duration(int64(adjustedInterval) * intervalFactor)
+ intervalFactor := model.IntervalFactor
+ if intervalFactor == 0 {
+ intervalFactor = 1
+ }
+
+ interval := time.Duration(int64(adjustedInterval) * intervalFactor)
+ intervalMs := int64(interval / time.Millisecond)
+ rangeS := query.TimeRange.To.Unix() - query.TimeRange.From.Unix()
+
+ // Interpolate variables in expr
+ expr := model.Expr
+ expr = strings.ReplaceAll(expr, "$__interval_ms", strconv.FormatInt(intervalMs, 10))
+ expr = strings.ReplaceAll(expr, "$__interval", intervalv2.FormatDuration(interval))
+ expr = strings.ReplaceAll(expr, "$__range_ms", strconv.FormatInt(rangeS*1000, 10))
+ expr = strings.ReplaceAll(expr, "$__range_s", strconv.FormatInt(rangeS, 10))
+ expr = strings.ReplaceAll(expr, "$__range", strconv.FormatInt(rangeS, 10)+"s")
+ expr = strings.ReplaceAll(expr, "$__rate_interval", intervalv2.FormatDuration(calculateRateInterval(interval, dsInfo.TimeInterval, s.intervalCalculator)))
qs = append(qs, &PrometheusQuery{
Expr: expr,
- Step: step,
- LegendFormat: format,
- Start: start,
- End: end,
- RefId: queryModel.RefID,
+ Step: interval,
+ LegendFormat: model.LegendFormat,
+ Start: query.TimeRange.From,
+ End: query.TimeRange.To,
+ RefId: query.RefID,
})
}
@@ -333,3 +340,18 @@ func ConvertAPIError(err error) error {
}
return err
}
+
+func calculateRateInterval(interval time.Duration, scrapeInterval string, intervalCalculator intervalv2.Calculator) time.Duration {
+ scrape := scrapeInterval
+ if scrape == "" {
+ scrape = "15s"
+ }
+
+ scrapeIntervalDuration, err := intervalv2.ParseIntervalStringToTimeDuration(scrape)
+ if err != nil {
+ return time.Duration(0)
+ }
+
+ rateInterval := time.Duration(int(math.Max(float64(interval+scrapeIntervalDuration), float64(4)*float64(scrapeIntervalDuration))))
+ return rateInterval
+}
diff --git a/pkg/tsdb/prometheus/prometheus_test.go b/pkg/tsdb/prometheus/prometheus_test.go
index 2518e9c8c2c..3293c167252 100644
--- a/pkg/tsdb/prometheus/prometheus_test.go
+++ b/pkg/tsdb/prometheus/prometheus_test.go
@@ -6,18 +6,14 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
- "github.com/grafana/grafana/pkg/tsdb"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
p "github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
)
var now = time.Now()
-func TestPrometheus(t *testing.T) {
- service := Service{
- intervalCalculator: tsdb.NewCalculator(),
- }
-
+func TestPrometheus_formatLeged(t *testing.T) {
t.Run("converting metric name", func(t *testing.T) {
metric := map[p.LabelName]p.LabelValue{
p.LabelName("app"): p.LabelValue("backend"),
@@ -44,161 +40,252 @@ func TestPrometheus(t *testing.T) {
require.Equal(t, `http_request_total{app="backend", device="mobile"}`, formatLegend(metric, query))
})
+}
+
+func TestPrometheus_parseQuery(t *testing.T) {
+ service := Service{
+ intervalCalculator: intervalv2.NewCalculator(),
+ }
+
+ t.Run("parsing query model with step", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(12 * time.Hour),
+ }
- t.Run("parsing query model with step and default stepMode", func(t *testing.T) {
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"refId": "A"
- }`)
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(12 * time.Hour),
- }
- query.TimeRange = timeRange
- models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Second*30, models[0].Step)
})
- t.Run("parsing query model with step and exact stepMode", func(t *testing.T) {
- query := queryContext(`{
- "expr": "go_goroutines",
- "format": "time_series",
- "refId": "A",
- "stepMode": "exact",
- "interval": "7s"
- }`)
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(12 * time.Hour),
- }
- query.TimeRange = timeRange
- models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
- require.NoError(t, err)
- require.Equal(t, time.Second*7, models[0].Step)
- })
-
- t.Run("parsing query model with short step and max stepMode", func(t *testing.T) {
- query := queryContext(`{
- "expr": "go_goroutines",
- "format": "time_series",
- "refId": "A",
- "stepMode": "max",
- "interval": "6s"
- }`)
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(12 * time.Hour),
- }
- query.TimeRange = timeRange
- models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
- require.NoError(t, err)
- require.Equal(t, time.Second*6, models[0].Step)
- })
-
- t.Run("parsing query model with long step and max stepMode", func(t *testing.T) {
- query := queryContext(`{
- "expr": "go_goroutines",
- "format": "time_series",
- "refId": "A",
- "stepMode": "max",
- "interval": "100s"
- }`)
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(12 * time.Hour),
- }
- query.TimeRange = timeRange
- models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
- require.NoError(t, err)
- require.Equal(t, time.Second*30, models[0].Step)
- })
-
- t.Run("parsing query model with unsafe interval", func(t *testing.T) {
- query := queryContext(`{
- "expr": "go_goroutines",
- "format": "time_series",
- "refId": "A",
- "stepMode": "max",
- "interval": "2s"
- }`)
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(12 * time.Hour),
- }
- query.TimeRange = timeRange
- models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
- require.NoError(t, err)
- require.Equal(t, time.Second*5, models[0].Step)
- })
-
t.Run("parsing query model without step parameter", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(1 * time.Hour),
+ }
+
query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
- }`)
- models, err := service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
- require.NoError(t, err)
- require.Equal(t, time.Minute*2, models[0].Step)
+ }`, timeRange)
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(1 * time.Hour),
- }
- query.TimeRange = timeRange
- models, err = service.parseQuery([]backend.DataQuery{query}, &DatasourceInfo{})
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Second*15, models[0].Step)
})
t.Run("parsing query model with high intervalFactor", func(t *testing.T) {
- models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 10,
"refId": "A"
- }`)}, &DatasourceInfo{})
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Minute*20, models[0].Step)
})
t.Run("parsing query model with low intervalFactor", func(t *testing.T) {
- models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
- }`)}, &DatasourceInfo{})
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Minute*2, models[0].Step)
})
t.Run("parsing query model specified scrape-interval in the data source", func(t *testing.T) {
- models, err := service.parseQuery([]backend.DataQuery{queryContext(`{
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
"expr": "go_goroutines",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
- }`)}, &DatasourceInfo{
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{
TimeInterval: "240s",
- })
+ }
+ models, err := service.parseQuery(query, dsInfo)
require.NoError(t, err)
require.Equal(t, time.Minute*4, models[0].Step)
})
+
+ t.Run("parsing query model with $__interval variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__interval]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [2m]})", models[0].Expr)
+ })
+
+ t.Run("parsing query model with $__interval_ms variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__interval_ms]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [120000]})", models[0].Expr)
+ })
+
+ t.Run("parsing query model with $__interval_ms and $__interval variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__interval_ms]}) + rate(ALERTS{job=\"test\" [$__interval]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [120000]}) + rate(ALERTS{job=\"test\" [2m]})", models[0].Expr)
+ })
+
+ t.Run("parsing query model with $__range variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__range]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [172800s]})", models[0].Expr)
+ })
+
+ t.Run("parsing query model with $__range_s variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__range_s]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [172800]})", models[0].Expr)
+ })
+
+ t.Run("parsing query model with $__range_ms variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(48 * time.Hour),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__range_ms]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [172800000]})", models[0].Expr)
+ })
+
+ t.Run("parsing query model with $__rate_interval variable", func(t *testing.T) {
+ timeRange := backend.TimeRange{
+ From: now,
+ To: now.Add(5 * time.Minute),
+ }
+
+ query := queryContext(`{
+ "expr": "rate(ALERTS{job=\"test\" [$__rate_interval]})",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }`, timeRange)
+
+ dsInfo := &DatasourceInfo{}
+ models, err := service.parseQuery(query, dsInfo)
+ require.NoError(t, err)
+ require.Equal(t, "rate(ALERTS{job=\"test\" [1m]})", models[0].Expr)
+ })
}
-func queryContext(json string) backend.DataQuery {
- timeRange := backend.TimeRange{
- From: now,
- To: now.Add(48 * time.Hour),
- }
- return backend.DataQuery{
- TimeRange: timeRange,
- RefID: "A",
- JSON: []byte(json),
+func queryContext(json string, timeRange backend.TimeRange) *backend.QueryDataRequest {
+ return &backend.QueryDataRequest{
+ Queries: []backend.DataQuery{
+ {
+ JSON: []byte(json),
+ TimeRange: timeRange,
+ RefID: "A",
+ },
+ },
}
}
diff --git a/pkg/tsdb/service.go b/pkg/tsdb/service.go
index 6f88b382c87..cdabac43b3c 100644
--- a/pkg/tsdb/service.go
+++ b/pkg/tsdb/service.go
@@ -4,31 +4,27 @@ import (
"context"
"fmt"
- "github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/cloudmonitoring"
- "github.com/grafana/grafana/pkg/tsdb/mssql"
- "github.com/grafana/grafana/pkg/tsdb/mysql"
- "github.com/grafana/grafana/pkg/tsdb/postgres"
+ _ "github.com/grafana/grafana/pkg/tsdb/postgres"
)
// NewService returns a new Service.
func NewService(
- cfg *setting.Cfg, pluginManager plugins.Manager, backendPluginManager backendplugin.Manager,
- oauthTokenService *oauthtoken.Service, httpClientProvider httpclient.Provider, cloudMonitoringService *cloudmonitoring.Service,
- postgresService *postgres.PostgresService,
+ cfg *setting.Cfg,
+ pluginManager plugins.Manager,
+ backendPluginManager backendplugin.Manager,
+ oauthTokenService *oauthtoken.Service,
+ cloudMonitoringService *cloudmonitoring.Service,
) *Service {
s := newService(cfg, pluginManager, backendPluginManager, oauthTokenService)
// register backend data sources using legacy plugin
// contracts/non-SDK contracts
- s.registry["mssql"] = mssql.NewExecutor
- s.registry["postgres"] = postgresService.NewExecutor
- s.registry["mysql"] = mysql.New(httpClientProvider)
s.registry["stackdriver"] = cloudMonitoringService.NewExecutor
return s
@@ -52,7 +48,6 @@ type Service struct {
PluginManager plugins.Manager
BackendPluginManager backendplugin.Manager
OAuthTokenService oauthtoken.OAuthTokenService
-
//nolint: staticcheck // plugins.DataPlugin deprecated
registry map[string]func(*models.DataSource) (plugins.DataPlugin, error)
}
@@ -70,7 +65,6 @@ func (s *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
return plugin.DataQuery(ctx, ds, query)
}
-
return dataPluginQueryAdapter(ds.Type, s.BackendPluginManager, s.OAuthTokenService).DataQuery(ctx, ds, query)
}
diff --git a/pkg/tsdb/service_test.go b/pkg/tsdb/service_test.go
index 86c66659bae..b8adf85f04f 100644
--- a/pkg/tsdb/service_test.go
+++ b/pkg/tsdb/service_test.go
@@ -142,6 +142,7 @@ func createService() (*Service, *fakeExecutor, *fakeBackendPM) {
manager := &manager.PluginManager{
BackendPluginManager: fakeBackendPM,
}
+
s := newService(setting.NewCfg(), manager, fakeBackendPM, &fakeOAuthTokenService{})
e := &fakeExecutor{
//nolint: staticcheck // plugins.DataPlugin deprecated
diff --git a/pkg/tsdb/sqleng/sql_engine.go b/pkg/tsdb/sqleng/sql_engine.go
index b3c505a52d9..f4745aa5cc6 100644
--- a/pkg/tsdb/sqleng/sql_engine.go
+++ b/pkg/tsdb/sqleng/sql_engine.go
@@ -3,6 +3,7 @@ package sqleng
import (
"context"
"database/sql"
+ "encoding/json"
"errors"
"fmt"
"net"
@@ -15,11 +16,9 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
- "github.com/grafana/grafana/pkg/plugins"
- "github.com/grafana/grafana/pkg/tsdb/interval"
+ "github.com/grafana/grafana/pkg/tsdb/intervalv2"
+ "github.com/grafana/grafana/pkg/util/errutil"
"xorm.io/core"
"xorm.io/xorm"
)
@@ -32,29 +31,28 @@ var ErrConnectionFailed = errors.New("failed to connect to server - please inspe
// SQLMacroEngine interpolates macros into sql. It takes in the Query to have access to query context and
// timeRange to be able to generate queries that use from and to.
type SQLMacroEngine interface {
- Interpolate(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, sql string) (string, error)
+ Interpolate(query *backend.DataQuery, timeRange backend.TimeRange, sql string) (string, error)
}
// SqlQueryResultTransformer transforms a query result row to RowValues with proper types.
type SqlQueryResultTransformer interface {
// TransformQueryError transforms a query error.
TransformQueryError(err error) error
-
GetConverterList() []sqlutil.StringConverter
}
type engineCacheType struct {
- cache map[int64]*xorm.Engine
- versions map[int64]int
+ cache map[int64]*xorm.Engine
+ updates map[int64]time.Time
sync.Mutex
}
var engineCache = engineCacheType{
- cache: make(map[int64]*xorm.Engine),
- versions: make(map[int64]int),
+ cache: make(map[int64]*xorm.Engine),
+ updates: make(map[int64]time.Time),
}
-var sqlIntervalCalculator = interval.NewCalculator()
+var sqlIntervalCalculator = intervalv2.NewCalculator()
// NewXormEngine is an xorm.Engine factory, that can be stubbed by tests.
//nolint:gocritic
@@ -62,24 +60,60 @@ var NewXormEngine = func(driverName string, connectionString string) (*xorm.Engi
return xorm.NewEngine(driverName, connectionString)
}
-type dataPlugin struct {
+type JsonData struct {
+ MaxOpenConns int `json:"maxOpenConns"`
+ MaxIdleConns int `json:"maxIdleConns"`
+ ConnMaxLifetime int `json:"connMaxLifetime"`
+ Timescaledb bool `json:"timescaledb"`
+ Mode string `json:"sslmode"`
+ ConfigurationMethod string `json:"tlsConfigurationMethod"`
+ RootCertFile string `json:"sslRootCertFile"`
+ CertFile string `json:"sslCertFile"`
+ CertKeyFile string `json:"sslKeyFile"`
+ Timezone string `json:"timezone"`
+ Encrypt string `json:"encrypt"`
+ TimeInterval string `json:"timeInterval"`
+}
+
+type DataSourceInfo struct {
+ JsonData JsonData
+ URL string
+ User string
+ Database string
+ ID int64
+ Updated time.Time
+ UID string
+ DecryptedSecureJSONData map[string]string
+}
+
+type DataPluginConfiguration struct {
+ DriverName string
+ DSInfo DataSourceInfo
+ ConnectionString string
+ TimeColumnNames []string
+ MetricColumnTypes []string
+ RowLimit int64
+}
+type DataSourceHandler struct {
macroEngine SQLMacroEngine
queryResultTransformer SqlQueryResultTransformer
engine *xorm.Engine
timeColumnNames []string
metricColumnTypes []string
log log.Logger
+ dsInfo DataSourceInfo
+ rowLimit int64
+}
+type QueryJson struct {
+ RawSql string `json:"rawSql"`
+ Fill bool `json:"fill"`
+ FillInterval float64 `json:"fillInterval"`
+ FillMode string `json:"fillMode"`
+ FillValue float64 `json:"fillValue"`
+ Format string `json:"format"`
}
-type DataPluginConfiguration struct {
- DriverName string
- Datasource *models.DataSource
- ConnectionString string
- TimeColumnNames []string
- MetricColumnTypes []string
-}
-
-func (e *dataPlugin) transformQueryError(err error) error {
+func (e *DataSourceHandler) transformQueryError(err error) error {
// OpError is the error type usually returned by functions in the net
// package. It describes the operation, network type, and address of
// an error. We log this error rather than return it to the client
@@ -93,32 +127,32 @@ func (e *dataPlugin) transformQueryError(err error) error {
return e.queryResultTransformer.TransformQueryError(err)
}
-// NewDataPlugin returns a new plugins.DataPlugin
-//nolint: staticcheck // plugins.DataPlugin deprecated
-func NewDataPlugin(config DataPluginConfiguration, queryResultTransformer SqlQueryResultTransformer,
- macroEngine SQLMacroEngine, log log.Logger) (plugins.DataPlugin, error) {
- plugin := dataPlugin{
+func NewQueryDataHandler(config DataPluginConfiguration, queryResultTransformer SqlQueryResultTransformer,
+ macroEngine SQLMacroEngine, log log.Logger) (*DataSourceHandler, error) {
+ queryDataHandler := DataSourceHandler{
queryResultTransformer: queryResultTransformer,
macroEngine: macroEngine,
timeColumnNames: []string{"time"},
log: log,
+ dsInfo: config.DSInfo,
+ rowLimit: config.RowLimit,
}
if len(config.TimeColumnNames) > 0 {
- plugin.timeColumnNames = config.TimeColumnNames
+ queryDataHandler.timeColumnNames = config.TimeColumnNames
}
if len(config.MetricColumnTypes) > 0 {
- plugin.metricColumnTypes = config.MetricColumnTypes
+ queryDataHandler.metricColumnTypes = config.MetricColumnTypes
}
engineCache.Lock()
defer engineCache.Unlock()
- if engine, present := engineCache.cache[config.Datasource.Id]; present {
- if version := engineCache.versions[config.Datasource.Id]; version == config.Datasource.Version {
- plugin.engine = engine
- return &plugin, nil
+ if engine, present := engineCache.cache[config.DSInfo.ID]; present {
+ if updateTime := engineCache.updates[config.DSInfo.ID]; updateTime.Before(config.DSInfo.Updated) {
+ queryDataHandler.engine = engine
+ return &queryDataHandler, nil
}
}
@@ -127,104 +161,102 @@ func NewDataPlugin(config DataPluginConfiguration, queryResultTransformer SqlQue
return nil, err
}
- maxOpenConns := config.Datasource.JsonData.Get("maxOpenConns").MustInt(0)
- engine.SetMaxOpenConns(maxOpenConns)
- maxIdleConns := config.Datasource.JsonData.Get("maxIdleConns").MustInt(2)
- engine.SetMaxIdleConns(maxIdleConns)
- connMaxLifetime := config.Datasource.JsonData.Get("connMaxLifetime").MustInt(14400)
- engine.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second)
+ engine.SetMaxOpenConns(config.DSInfo.JsonData.MaxOpenConns)
+ engine.SetMaxIdleConns(config.DSInfo.JsonData.MaxIdleConns)
+ engine.SetConnMaxLifetime(time.Duration(config.DSInfo.JsonData.ConnMaxLifetime) * time.Second)
- engineCache.versions[config.Datasource.Id] = config.Datasource.Version
- engineCache.cache[config.Datasource.Id] = engine
- plugin.engine = engine
-
- return &plugin, nil
+ engineCache.updates[config.DSInfo.ID] = config.DSInfo.Updated
+ engineCache.cache[config.DSInfo.ID] = engine
+ queryDataHandler.engine = engine
+ return &queryDataHandler, nil
}
-const rowLimit = 1000000
+type DBDataResponse struct {
+ dataResponse backend.DataResponse
+ refID string
+}
-// DataQuery queries for data.
-//nolint: staticcheck // plugins.DataPlugin deprecated
-func (e *dataPlugin) DataQuery(ctx context.Context, dsInfo *models.DataSource,
- queryContext plugins.DataQuery) (plugins.DataResponse, error) {
- ch := make(chan plugins.DataQueryResult, len(queryContext.Queries))
+func (e *DataSourceHandler) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+ result := backend.NewQueryDataResponse()
+ ch := make(chan DBDataResponse, len(req.Queries))
var wg sync.WaitGroup
// Execute each query in a goroutine and wait for them to finish afterwards
- for _, query := range queryContext.Queries {
- if query.Model.Get("rawSql").MustString() == "" {
+ for _, query := range req.Queries {
+ queryjson := QueryJson{
+ Fill: false,
+ Format: "time_series",
+ }
+ err := json.Unmarshal(query.JSON, &queryjson)
+ if err != nil {
+ return nil, fmt.Errorf("error unmarshal query json: %w", err)
+ }
+ if queryjson.RawSql == "" {
continue
}
wg.Add(1)
- go e.executeQuery(query, &wg, queryContext, ch)
+ go e.executeQuery(query, &wg, ctx, ch, queryjson)
}
wg.Wait()
// Read results from channels
close(ch)
- result := plugins.DataResponse{
- Results: make(map[string]plugins.DataQueryResult),
- }
+ result.Responses = make(map[string]backend.DataResponse)
for queryResult := range ch {
- result.Results[queryResult.RefID] = queryResult
+ result.Responses[queryResult.refID] = queryResult.dataResponse
}
return result, nil
}
-//nolint: staticcheck,gocyclo // plugins.DataQueryResult deprecated
-func (e *dataPlugin) executeQuery(query plugins.DataSubQuery, wg *sync.WaitGroup, queryContext plugins.DataQuery,
- ch chan plugins.DataQueryResult) {
+func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitGroup, queryContext context.Context,
+ ch chan DBDataResponse, queryJson QueryJson) {
defer wg.Done()
-
- queryResult := plugins.DataQueryResult{
- Meta: simplejson.New(),
- RefID: query.RefID,
+ queryResult := DBDataResponse{
+ dataResponse: backend.DataResponse{},
+ refID: query.RefID,
}
defer func() {
if r := recover(); r != nil {
e.log.Error("executeQuery panic", "error", r, "stack", log.Stack(1))
if theErr, ok := r.(error); ok {
- queryResult.Error = theErr
+ queryResult.dataResponse.Error = theErr
} else if theErrString, ok := r.(string); ok {
- queryResult.Error = fmt.Errorf(theErrString)
+ queryResult.dataResponse.Error = fmt.Errorf(theErrString)
} else {
- queryResult.Error = fmt.Errorf("unexpected error, see the server log for details")
+ queryResult.dataResponse.Error = fmt.Errorf("unexpected error, see the server log for details")
}
ch <- queryResult
}
}()
- rawSQL := query.Model.Get("rawSql").MustString()
- if rawSQL == "" {
+ if queryJson.RawSql == "" {
panic("Query model property rawSql should not be empty at this point")
}
- var timeRange plugins.DataTimeRange
- if queryContext.TimeRange != nil {
- timeRange = *queryContext.TimeRange
- }
+
+ timeRange := query.TimeRange
errAppendDebug := func(frameErr string, err error, query string) {
var emptyFrame data.Frame
emptyFrame.SetMeta(&data.FrameMeta{
ExecutedQueryString: query,
})
- queryResult.Error = fmt.Errorf("%s: %w", frameErr, err)
- queryResult.Dataframes = plugins.NewDecodedDataFrames(data.Frames{&emptyFrame})
+ queryResult.dataResponse.Error = fmt.Errorf("%s: %w", frameErr, err)
+ queryResult.dataResponse.Frames = data.Frames{&emptyFrame}
ch <- queryResult
}
// global substitutions
- interpolatedQuery, err := Interpolate(query, timeRange, rawSQL)
+ interpolatedQuery, err := Interpolate(query, timeRange, e.dsInfo.JsonData.TimeInterval, queryJson.RawSql)
if err != nil {
errAppendDebug("interpolation failed", e.transformQueryError(err), interpolatedQuery)
return
}
// data source specific substitutions
- interpolatedQuery, err = e.macroEngine.Interpolate(query, timeRange, interpolatedQuery)
+ interpolatedQuery, err = e.macroEngine.Interpolate(&query, timeRange, interpolatedQuery)
if err != nil {
errAppendDebug("interpolation failed", e.transformQueryError(err), interpolatedQuery)
return
@@ -253,28 +285,28 @@ func (e *dataPlugin) executeQuery(query plugins.DataSubQuery, wg *sync.WaitGroup
// Convert row.Rows to dataframe
stringConverters := e.queryResultTransformer.GetConverterList()
- frame, err := sqlutil.FrameFromRows(rows.Rows, rowLimit, sqlutil.ToConverters(stringConverters...)...)
+ frame, err := sqlutil.FrameFromRows(rows.Rows, e.rowLimit, sqlutil.ToConverters(stringConverters...)...)
if err != nil {
errAppendDebug("convert frame from rows error", err, interpolatedQuery)
return
}
- frame.SetMeta(&data.FrameMeta{
- ExecutedQueryString: interpolatedQuery,
- })
+ if frame.Meta == nil {
+ frame.Meta = &data.FrameMeta{}
+ }
+
+ frame.Meta.ExecutedQueryString = interpolatedQuery
// If no rows were returned, no point checking anything else.
if frame.Rows() == 0 {
- queryResult.Dataframes = plugins.NewDecodedDataFrames(data.Frames{frame})
+ queryResult.dataResponse.Frames = data.Frames{frame}
ch <- queryResult
return
}
- if qm.timeIndex != -1 {
- if err := convertSQLTimeColumnToEpochMS(frame, qm.timeIndex); err != nil {
- errAppendDebug("db convert time column failed", err, interpolatedQuery)
- return
- }
+ if err := convertSQLTimeColumnsToEpochMS(frame, qm); err != nil {
+ errAppendDebug("converting time columns failed", err, interpolatedQuery)
+ return
}
if qm.Format == dataQueryFormatSeries {
@@ -338,31 +370,28 @@ func (e *dataPlugin) executeQuery(query plugins.DataSubQuery, wg *sync.WaitGroup
}
}
- queryResult.Dataframes = plugins.NewDecodedDataFrames(data.Frames{frame})
+ queryResult.dataResponse.Frames = data.Frames{frame}
ch <- queryResult
}
// Interpolate provides global macros/substitutions for all sql datasources.
-var Interpolate = func(query plugins.DataSubQuery, timeRange plugins.DataTimeRange, sql string) (string, error) {
- minInterval, err := interval.GetIntervalFrom(query.DataSource, query.Model, time.Second*60)
- if err != nil {
- return "", err
- }
- interval, err := sqlIntervalCalculator.Calculate(timeRange, minInterval, "min")
+var Interpolate = func(query backend.DataQuery, timeRange backend.TimeRange, timeInterval string, sql string) (string, error) {
+ minInterval, err := intervalv2.GetIntervalFrom(timeInterval, query.Interval.String(), query.Interval.Milliseconds(), time.Second*60)
if err != nil {
return "", err
}
+ interval := sqlIntervalCalculator.Calculate(timeRange, minInterval, query.MaxDataPoints)
sql = strings.ReplaceAll(sql, "$__interval_ms", strconv.FormatInt(interval.Milliseconds(), 10))
sql = strings.ReplaceAll(sql, "$__interval", interval.Text)
- sql = strings.ReplaceAll(sql, "$__unixEpochFrom()", fmt.Sprintf("%d", timeRange.GetFromAsSecondsEpoch()))
- sql = strings.ReplaceAll(sql, "$__unixEpochTo()", fmt.Sprintf("%d", timeRange.GetToAsSecondsEpoch()))
+ sql = strings.ReplaceAll(sql, "$__unixEpochFrom()", fmt.Sprintf("%d", timeRange.From.UTC().Unix()))
+ sql = strings.ReplaceAll(sql, "$__unixEpochTo()", fmt.Sprintf("%d", timeRange.To.UTC().Unix()))
return sql, nil
}
//nolint: staticcheck // plugins.DataPlugin deprecated
-func (e *dataPlugin) newProcessCfg(query plugins.DataSubQuery, queryContext plugins.DataQuery,
+func (e *DataSourceHandler) newProcessCfg(query backend.DataQuery, queryContext context.Context,
rows *core.Rows, interpolatedQuery string) (*dataQueryModel, error) {
columnNames, err := rows.Columns()
if err != nil {
@@ -378,40 +407,44 @@ func (e *dataPlugin) newProcessCfg(query plugins.DataSubQuery, queryContext plug
columnNames: columnNames,
rows: rows,
timeIndex: -1,
+ timeEndIndex: -1,
metricIndex: -1,
metricPrefix: false,
queryContext: queryContext,
}
- if query.Model.Get("fill").MustBool(false) {
+ queryJson := QueryJson{}
+ err = json.Unmarshal(query.JSON, &queryJson)
+ if err != nil {
+ return nil, err
+ }
+
+ if queryJson.Fill {
qm.FillMissing = &data.FillMissing{}
- qm.Interval = time.Duration(query.Model.Get("fillInterval").MustFloat64() * float64(time.Second))
- switch strings.ToLower(query.Model.Get("fillMode").MustString()) {
+ qm.Interval = time.Duration(queryJson.FillInterval * float64(time.Second))
+ switch strings.ToLower(queryJson.FillMode) {
case "null":
qm.FillMissing.Mode = data.FillModeNull
case "previous":
qm.FillMissing.Mode = data.FillModePrevious
case "value":
qm.FillMissing.Mode = data.FillModeValue
- qm.FillMissing.Value = query.Model.Get("fillValue").MustFloat64()
+ qm.FillMissing.Value = queryJson.FillValue
default:
}
}
//nolint: staticcheck // plugins.DataPlugin deprecated
- if queryContext.TimeRange != nil {
- qm.TimeRange.From = queryContext.TimeRange.GetFromAsTimeUTC()
- qm.TimeRange.To = queryContext.TimeRange.GetToAsTimeUTC()
- }
+ qm.TimeRange.From = query.TimeRange.From.UTC()
+ qm.TimeRange.To = query.TimeRange.To.UTC()
- format := query.Model.Get("format").MustString("time_series")
- switch format {
+ switch queryJson.Format {
case "time_series":
qm.Format = dataQueryFormatSeries
case "table":
qm.Format = dataQueryFormatTable
default:
- panic(fmt.Sprintf("Unrecognized query model format: %q", format))
+ panic(fmt.Sprintf("Unrecognized query model format: %q", queryJson.Format))
}
for i, col := range qm.columnNames {
@@ -421,6 +454,12 @@ func (e *dataPlugin) newProcessCfg(query plugins.DataSubQuery, queryContext plug
break
}
}
+
+ if qm.Format == dataQueryFormatTable && col == "timeend" {
+ qm.timeEndIndex = i
+ continue
+ }
+
switch col {
case "metric":
qm.metricIndex = i
@@ -459,10 +498,11 @@ type dataQueryModel struct {
columnNames []string
columnTypes []*sql.ColumnType
timeIndex int
+ timeEndIndex int
metricIndex int
rows *core.Rows
metricPrefix bool
- queryContext plugins.DataQuery
+ queryContext context.Context
}
func convertInt64ToFloat64(origin *data.Field, newField *data.Field) {
@@ -788,6 +828,22 @@ func convertNullableFloat32ToEpochMS(origin *data.Field, newField *data.Field) {
}
}
+func convertSQLTimeColumnsToEpochMS(frame *data.Frame, qm *dataQueryModel) error {
+ if qm.timeIndex != -1 {
+ if err := convertSQLTimeColumnToEpochMS(frame, qm.timeIndex); err != nil {
+ return errutil.Wrap("failed to convert time column", err)
+ }
+ }
+
+ if qm.timeEndIndex != -1 {
+ if err := convertSQLTimeColumnToEpochMS(frame, qm.timeEndIndex); err != nil {
+ return errutil.Wrap("failed to convert timeend column", err)
+ }
+ }
+
+ return nil
+}
+
// convertSQLTimeColumnToEpochMS converts column named time to unix timestamp in milliseconds
// to make native datetime types and epoch dates work in annotation and table queries.
func convertSQLTimeColumnToEpochMS(frame *data.Frame, timeIndex int) error {
@@ -902,23 +958,36 @@ func convertSQLValueColumnToFloat(frame *data.Frame, Index int) (*data.Frame, er
return frame, nil
}
-func SetupFillmode(query plugins.DataSubQuery, interval time.Duration, fillmode string) error {
- query.Model.Set("fill", true)
- query.Model.Set("fillInterval", interval.Seconds())
+func SetupFillmode(query *backend.DataQuery, interval time.Duration, fillmode string) error {
+ rawQueryProp := make(map[string]interface{})
+ queryBytes, err := query.JSON.MarshalJSON()
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(queryBytes, &rawQueryProp)
+ if err != nil {
+ return err
+ }
+ rawQueryProp["fill"] = true
+ rawQueryProp["fillInterval"] = interval.Seconds()
+
switch fillmode {
case "NULL":
- query.Model.Set("fillMode", "null")
+ rawQueryProp["fillMode"] = "null"
case "previous":
- query.Model.Set("fillMode", "previous")
+ rawQueryProp["fillMode"] = "previous"
default:
- query.Model.Set("fillMode", "value")
+ rawQueryProp["fillMode"] = "value"
floatVal, err := strconv.ParseFloat(fillmode, 64)
if err != nil {
return fmt.Errorf("error parsing fill value %v", fillmode)
}
- query.Model.Set("fillValue", floatVal)
+ rawQueryProp["fillValue"] = floatVal
+ }
+ query.JSON, err = json.Marshal(rawQueryProp)
+ if err != nil {
+ return err
}
-
return nil
}
diff --git a/pkg/tsdb/sqleng/sql_engine_test.go b/pkg/tsdb/sqleng/sql_engine_test.go
index 2d926cb27d5..eafe751fe6f 100644
--- a/pkg/tsdb/sqleng/sql_engine_test.go
+++ b/pkg/tsdb/sqleng/sql_engine_test.go
@@ -7,11 +7,10 @@ import (
"testing"
"time"
+ "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
- "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
- "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -25,35 +24,35 @@ func TestSQLEngine(t *testing.T) {
t.Run("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func(t *testing.T) {
from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC)
to := from.Add(5 * time.Minute)
- timeRange := plugins.DataTimeRange{From: "5m", To: "now", Now: to}
- query := plugins.DataSubQuery{DataSource: &models.DataSource{}, Model: simplejson.New()}
+ timeRange := backend.TimeRange{From: from, To: to}
+ query := backend.DataQuery{JSON: []byte("{}")}
t.Run("interpolate $__interval", func(t *testing.T) {
- sql, err := Interpolate(query, timeRange, "select $__interval ")
+ sql, err := Interpolate(query, timeRange, "", "select $__interval ")
require.NoError(t, err)
require.Equal(t, "select 1m ", sql)
})
t.Run("interpolate $__interval in $__timeGroup", func(t *testing.T) {
- sql, err := Interpolate(query, timeRange, "select $__timeGroupAlias(time,$__interval)")
+ sql, err := Interpolate(query, timeRange, "", "select $__timeGroupAlias(time,$__interval)")
require.NoError(t, err)
require.Equal(t, "select $__timeGroupAlias(time,1m)", sql)
})
t.Run("interpolate $__interval_ms", func(t *testing.T) {
- sql, err := Interpolate(query, timeRange, "select $__interval_ms ")
+ sql, err := Interpolate(query, timeRange, "", "select $__interval_ms ")
require.NoError(t, err)
require.Equal(t, "select 60000 ", sql)
})
t.Run("interpolate __unixEpochFrom function", func(t *testing.T) {
- sql, err := Interpolate(query, timeRange, "select $__unixEpochFrom()")
+ sql, err := Interpolate(query, timeRange, "", "select $__unixEpochFrom()")
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("select %d", from.Unix()), sql)
})
t.Run("interpolate __unixEpochTo function", func(t *testing.T) {
- sql, err := Interpolate(query, timeRange, "select $__unixEpochTo()")
+ sql, err := Interpolate(query, timeRange, "", "select $__unixEpochTo()")
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("select %d", to.Unix()), sql)
})
@@ -406,7 +405,7 @@ func TestSQLEngine(t *testing.T) {
for _, tc := range tests {
transformer := &testQueryResultTransformer{}
- dp := dataPlugin{
+ dp := DataSourceHandler{
log: log.New("test"),
queryResultTransformer: transformer,
}
diff --git a/pkg/tsdb/testdatasource/csv_data.go b/pkg/tsdb/testdatasource/csv_data.go
index ee8bd133dea..584663453b3 100644
--- a/pkg/tsdb/testdatasource/csv_data.go
+++ b/pkg/tsdb/testdatasource/csv_data.go
@@ -30,7 +30,7 @@ func (p *TestDataPlugin) handleCsvContentScenario(ctx context.Context, req *back
csvContent := model.Get("csvContent").MustString()
alias := model.Get("alias").MustString("")
- frame, err := p.loadCsvContent(strings.NewReader(csvContent), alias)
+ frame, err := LoadCsvContent(strings.NewReader(csvContent), alias)
if err != nil {
return nil, err
}
@@ -94,10 +94,11 @@ func (p *TestDataPlugin) loadCsvFile(fileName string) (*data.Frame, error) {
}
}()
- return p.loadCsvContent(fileReader, fileName)
+ return LoadCsvContent(fileReader, fileName)
}
-func (p *TestDataPlugin) loadCsvContent(ioReader io.Reader, name string) (*data.Frame, error) {
+// LoadCsvContent should be moved to the SDK
+func LoadCsvContent(ioReader io.Reader, name string) (*data.Frame, error) {
reader := csv.NewReader(ioReader)
// Read the header records
diff --git a/pkg/tsdb/testdatasource/csv_data_test.go b/pkg/tsdb/testdatasource/csv_data_test.go
index 316a8a96546..5226e3f8c8e 100644
--- a/pkg/tsdb/testdatasource/csv_data_test.go
+++ b/pkg/tsdb/testdatasource/csv_data_test.go
@@ -35,7 +35,7 @@ func TestCSVFileScenario(t *testing.T) {
_ = fileReader.Close()
}()
- frame, err := p.loadCsvContent(fileReader, name)
+ frame, err := LoadCsvContent(fileReader, name)
require.NoError(t, err)
require.NotNil(t, frame)
diff --git a/pkg/tsdb/testdatasource/scenarios.go b/pkg/tsdb/testdatasource/scenarios.go
index fde7981773e..b07e696ccb1 100644
--- a/pkg/tsdb/testdatasource/scenarios.go
+++ b/pkg/tsdb/testdatasource/scenarios.go
@@ -268,7 +268,7 @@ func (p *TestDataPlugin) handleRandomWalkScenario(ctx context.Context, req *back
for i := 0; i < seriesCount; i++ {
respD := resp.Responses[q.RefID]
- respD.Frames = append(respD.Frames, randomWalk(q, model, i))
+ respD.Frames = append(respD.Frames, RandomWalk(q, model, i))
resp.Responses[q.RefID] = respD
}
}
@@ -354,7 +354,7 @@ func (p *TestDataPlugin) handleRandomWalkWithErrorScenario(ctx context.Context,
}
respD := resp.Responses[q.RefID]
- respD.Frames = append(respD.Frames, randomWalk(q, model, 0))
+ respD.Frames = append(respD.Frames, RandomWalk(q, model, 0))
respD.Error = fmt.Errorf("this is an error and it can include URLs http://grafana.com/")
resp.Responses[q.RefID] = respD
}
@@ -376,7 +376,7 @@ func (p *TestDataPlugin) handleRandomWalkSlowScenario(ctx context.Context, req *
time.Sleep(parsedInterval)
respD := resp.Responses[q.RefID]
- respD.Frames = append(respD.Frames, randomWalk(q, model, 0))
+ respD.Frames = append(respD.Frames, RandomWalk(q, model, 0))
resp.Responses[q.RefID] = respD
}
@@ -618,7 +618,7 @@ func (p *TestDataPlugin) handleLogsScenario(ctx context.Context, req *backend.Qu
return resp, nil
}
-func randomWalk(query backend.DataQuery, model *simplejson.Json, index int) *data.Frame {
+func RandomWalk(query backend.DataQuery, model *simplejson.Json, index int) *data.Frame {
timeWalkerMs := query.TimeRange.From.UnixNano() / int64(time.Millisecond)
to := query.TimeRange.To.UnixNano() / int64(time.Millisecond)
startValue := model.Get("startValue").MustFloat64(rand.Float64() * 100)
diff --git a/pkg/tsdb/testdatasource/stream_handler.go b/pkg/tsdb/testdatasource/stream_handler.go
index be610ff9a04..3d229dc846a 100644
--- a/pkg/tsdb/testdatasource/stream_handler.go
+++ b/pkg/tsdb/testdatasource/stream_handler.go
@@ -16,9 +16,11 @@ import (
type testStreamHandler struct {
logger log.Logger
frame *data.Frame
+ // If Live Pipeline enabled we are sending the whole frame to have a chance to process stream with rules.
+ livePipelineEnabled bool
}
-func newTestStreamHandler(logger log.Logger) *testStreamHandler {
+func newTestStreamHandler(logger log.Logger, livePipelineEnabled bool) *testStreamHandler {
frame := data.NewFrame("testdata",
data.NewField("Time", nil, make([]time.Time, 1)),
data.NewField("Value", nil, make([]float64, 1)),
@@ -26,8 +28,9 @@ func newTestStreamHandler(logger log.Logger) *testStreamHandler {
data.NewField("Max", nil, make([]float64, 1)),
)
return &testStreamHandler{
- frame: frame,
- logger: logger,
+ frame: frame,
+ logger: logger,
+ livePipelineEnabled: livePipelineEnabled,
}
}
@@ -117,9 +120,14 @@ func (p *testStreamHandler) runTestStream(ctx context.Context, path string, conf
continue
}
+ mode := data.IncludeDataOnly
+ if p.livePipelineEnabled {
+ mode = data.IncludeAll
+ }
+
if flight != nil {
flight.set(0, conf.Flight.getNextPoint(t))
- if err := sender.SendFrame(flight.frame, data.IncludeDataOnly); err != nil {
+ if err := sender.SendFrame(flight.frame, mode); err != nil {
return err
}
} else {
@@ -130,7 +138,7 @@ func (p *testStreamHandler) runTestStream(ctx context.Context, path string, conf
p.frame.Fields[1].Set(0, walker) // Value
p.frame.Fields[2].Set(0, walker-((rand.Float64()*spread)+0.01)) // Min
p.frame.Fields[3].Set(0, walker+((rand.Float64()*spread)+0.01)) // Max
- if err := sender.SendFrame(p.frame, data.IncludeDataOnly); err != nil {
+ if err := sender.SendFrame(p.frame, mode); err != nil {
return err
}
}
diff --git a/pkg/tsdb/testdatasource/testdata.go b/pkg/tsdb/testdatasource/testdata.go
index a4af2fec50f..60506f8a479 100644
--- a/pkg/tsdb/testdatasource/testdata.go
+++ b/pkg/tsdb/testdatasource/testdata.go
@@ -18,7 +18,7 @@ func ProvideService(cfg *setting.Cfg, manager backendplugin.Manager) (*TestDataP
factory := coreplugin.New(backend.ServeOpts{
QueryDataHandler: p.queryMux,
CallResourceHandler: httpadapter.New(resourceMux),
- StreamHandler: newTestStreamHandler(p.logger),
+ StreamHandler: newTestStreamHandler(p.logger, cfg.FeatureToggles["live-pipeline"]),
})
err := manager.Register("testdata", factory)
if err != nil {
diff --git a/pkg/util/shortid_generator.go b/pkg/util/shortid_generator.go
index c035e088944..767ea1a9f3e 100644
--- a/pkg/util/shortid_generator.go
+++ b/pkg/util/shortid_generator.go
@@ -20,6 +20,11 @@ func IsValidShortUID(uid string) bool {
return validUIDPattern(uid)
}
+// IsShortUIDTooLong checks if short unique identifier is too long
+func IsShortUIDTooLong(uid string) bool {
+ return len(uid) > 40
+}
+
// GenerateShortUID generates a short unique identifier.
func GenerateShortUID() string {
return shortid.MustGenerate()
diff --git a/pkg/util/shortid_generator_test.go b/pkg/util/shortid_generator_test.go
index 94055f0bcfd..2e0fc0cc6d7 100644
--- a/pkg/util/shortid_generator_test.go
+++ b/pkg/util/shortid_generator_test.go
@@ -1,6 +1,10 @@
package util
-import "testing"
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
func TestAllowedCharMatchesUidPattern(t *testing.T) {
for _, c := range allowedChars {
@@ -9,3 +13,33 @@ func TestAllowedCharMatchesUidPattern(t *testing.T) {
}
}
}
+
+func TestIsShortUIDTooLong(t *testing.T) {
+ var tests = []struct {
+ name string
+ uid string
+ expected bool
+ }{
+ {
+ name: "when the length of uid is longer than 40 chars then IsShortUIDTooLong should return true",
+ uid: allowedChars,
+ expected: true,
+ },
+ {
+ name: "when the length of uid is equal too 40 chars then IsShortUIDTooLong should return false",
+ uid: "0123456789012345678901234567890123456789",
+ expected: false,
+ },
+ {
+ name: "when the length of uid is shorter than 40 chars then IsShortUIDTooLong should return false",
+ uid: "012345678901234567890123456789012345678",
+ expected: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ require.Equal(t, tt.expected, IsShortUIDTooLong(tt.uid))
+ })
+ }
+}
diff --git a/public/app/app.ts b/public/app/app.ts
index 6ea299b0da4..58fa46d24b8 100644
--- a/public/app/app.ts
+++ b/public/app/app.ts
@@ -31,7 +31,7 @@ import { reportPerformance } from './core/services/echo/EchoSrv';
import { PerformanceBackend } from './core/services/echo/backends/PerformanceBackend';
import 'app/routes/GrafanaCtrl';
import 'app/features/all';
-import { getScrollbarWidth, getStandardFieldConfigs, getStandardOptionEditors } from '@grafana/ui';
+import { getScrollbarWidth, getStandardFieldConfigs } from '@grafana/ui';
import { getDefaultVariableAdapters, variableAdapters } from './features/variables/adapters';
import { initDevFeatures } from './dev';
import { getStandardTransformers } from 'app/core/utils/standardTransformers';
@@ -49,6 +49,7 @@ import getDefaultMonacoLanguages from '../lib/monaco-languages';
import { contextSrv } from './core/services/context_srv';
import { GAEchoBackend } from './core/services/echo/backends/analytics/GABackend';
import { RudderstackBackend } from './core/services/echo/backends/analytics/RudderstackBackend';
+import { getAllOptionEditors } from './core/components/editors/registry';
// add move to lodash for backward compatabilty with plugins
// @ts-ignore
@@ -81,7 +82,7 @@ export class GrafanaApp {
initExtensions();
configureStore();
- standardEditorsRegistry.setInit(getStandardOptionEditors);
+ standardEditorsRegistry.setInit(getAllOptionEditors);
standardFieldConfigEditorRegistry.setInit(getStandardFieldConfigs);
standardTransformersRegistry.setInit(getStandardTransformers);
variableAdapters.setInit(getDefaultVariableAdapters);
diff --git a/public/app/core/components/TraceToLogsSettings.tsx b/public/app/core/components/TraceToLogsSettings.tsx
index 742cd8e994f..64ab1668c17 100644
--- a/public/app/core/components/TraceToLogsSettings.tsx
+++ b/public/app/core/components/TraceToLogsSettings.tsx
@@ -6,7 +6,7 @@ import {
updateDatasourcePluginJsonDataOption,
} from '@grafana/data';
import { DataSourcePicker } from '@grafana/runtime';
-import { InlineField, InlineFieldRow, Input, TagsInput, useStyles } from '@grafana/ui';
+import { InlineField, InlineFieldRow, Input, TagsInput, useStyles, InlineSwitch } from '@grafana/ui';
import React from 'react';
export interface TraceToLogsOptions {
@@ -14,6 +14,8 @@ export interface TraceToLogsOptions {
tags?: string[];
spanStartTimeShift?: string;
spanEndTimeShift?: string;
+ filterByTraceID?: boolean;
+ filterBySpanID?: boolean;
}
export interface TraceToLogsData extends DataSourceJsonData {
@@ -112,6 +114,44 @@ export function TraceToLogsSettings({ options, onOptionsChange }: Props) {
/>
+
+
+
+ ) =>
+ updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToLogs', {
+ ...options.jsonData.tracesToLogs,
+ filterByTraceID: event.currentTarget.checked,
+ })
+ }
+ />
+
+
+
+
+
+ ) =>
+ updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToLogs', {
+ ...options.jsonData.tracesToLogs,
+ filterBySpanID: event.currentTarget.checked,
+ })
+ }
+ />
+
+
+ - Single frame
+ - 1st field is time field
+ - Time in ascending order, but may have duplictes
+ - String values are represented as separate fields rather than as labels
+ - Multiple value fields may exist
+
+ ),
+};
+
+const formats: Array> = [wideInfo, manyInfo, longInfo];
export function PrepareTimeSeriesEditor(props: TransformerUIProps): React.ReactElement {
const { options, onChange } = props;
@@ -64,9 +80,7 @@ export function PrepareTimeSeriesEditor(props: TransformerUIProps
-
- {options.format === timeSeriesFormat.TimeSeriesMany ? manyInfo.info : wideInfo.info}
-
+ {(formats.find((v) => v.value === options.format) || formats[0]).info}
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
index ac7012f879e..059a0aed15c 100644
--- a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts
@@ -6,18 +6,19 @@ import {
toDataFrameDTO,
DataFrameDTO,
DataFrameType,
+ getFrameDisplayName,
} from '@grafana/data';
import { prepareTimeSeriesTransformer, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
-describe('Prepair time series transformer', () => {
+describe('Prepare time series transformer', () => {
it('should transform wide to many', () => {
const source = [
toDataFrame({
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
{ name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
],
}),
@@ -32,8 +33,8 @@ describe('Prepair time series transformer', () => {
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
],
meta: {
type: DataFrameType.TimeSeriesMany,
@@ -44,7 +45,7 @@ describe('Prepair time series transformer', () => {
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+ { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
{ name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
],
meta: {
@@ -55,16 +56,16 @@ describe('Prepair time series transformer', () => {
]);
});
- it('should remove string fields since time series format is expected to be time/number fields', () => {
+ it('should treat string fields as labels', () => {
const source = [
toDataFrame({
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
- { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+ { name: 'time', type: FieldType.time, values: [1, 1, 2, 2] },
+ { name: 'region', type: FieldType.string, values: ['a', 'b', 'a', 'b'] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40] },
+ { name: 'more', type: FieldType.number, values: [2, 3, 4, 5] },
],
}),
];
@@ -73,32 +74,75 @@ describe('Prepair time series transformer', () => {
format: timeSeriesFormat.TimeSeriesMany,
};
- expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([
- toEquableDataFrame({
- name: 'wide',
- refId: 'A',
- fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
- ],
- length: 6,
- meta: {
- type: DataFrameType.TimeSeriesMany,
+ const frames = prepareTimeSeriesTransformer.transformer(config)(source);
+ expect(frames.length).toEqual(4);
+ expect(
+ frames.map((f) => ({
+ name: getFrameDisplayName(f),
+ labels: f.fields[1].labels,
+ time: f.fields[0].values.toArray(),
+ values: f.fields[1].values.toArray(),
+ }))
+ ).toMatchInlineSnapshot(`
+ Array [
+ Object {
+ "labels": Object {
+ "region": "a",
+ },
+ "name": "wide",
+ "time": Array [
+ 1,
+ 2,
+ ],
+ "values": Array [
+ 10,
+ 30,
+ ],
},
- }),
- toEquableDataFrame({
- name: 'wide',
- refId: 'A',
- fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
- ],
- length: 6,
- meta: {
- type: DataFrameType.TimeSeriesMany,
+ Object {
+ "labels": Object {
+ "region": "b",
+ },
+ "name": "wide",
+ "time": Array [
+ 1,
+ 2,
+ ],
+ "values": Array [
+ 20,
+ 40,
+ ],
},
- }),
- ]);
+ Object {
+ "labels": Object {
+ "region": "a",
+ },
+ "name": "wide",
+ "time": Array [
+ 1,
+ 2,
+ ],
+ "values": Array [
+ 2,
+ 4,
+ ],
+ },
+ Object {
+ "labels": Object {
+ "region": "b",
+ },
+ "name": "wide",
+ "time": Array [
+ 1,
+ 2,
+ ],
+ "values": Array [
+ 3,
+ 5,
+ ],
+ },
+ ]
+ `);
});
it('should transform all wide to many when mixed', () => {
@@ -107,9 +151,8 @@ describe('Prepair time series transformer', () => {
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
{ name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
],
}),
@@ -117,7 +160,7 @@ describe('Prepair time series transformer', () => {
name: 'long',
refId: 'B',
fields: [
- { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+ { name: 'time', type: FieldType.time, values: [4, 5, 6, 7, 8, 9] },
{ name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
],
}),
@@ -132,8 +175,8 @@ describe('Prepair time series transformer', () => {
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] },
+ { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
],
length: 6,
meta: {
@@ -144,8 +187,8 @@ describe('Prepair time series transformer', () => {
name: 'wide',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+ { name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
],
length: 6,
meta: {
@@ -156,7 +199,7 @@ describe('Prepair time series transformer', () => {
name: 'long',
refId: 'B',
fields: [
- { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+ { name: 'time', type: FieldType.time, values: [4, 5, 6, 7, 8, 9] },
{ name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
],
length: 6,
@@ -173,16 +216,16 @@ describe('Prepair time series transformer', () => {
name: 'long',
refId: 'A',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
],
}),
toDataFrame({
name: 'long',
refId: 'B',
fields: [
- { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
- { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
+ { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
],
}),
];
@@ -231,6 +274,52 @@ describe('Prepair time series transformer', () => {
expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([]);
});
+
+ it('should convert long to many', () => {
+ const source = [
+ toDataFrame({
+ name: 'long',
+ refId: 'X',
+ fields: [
+ { name: 'time', type: FieldType.time, values: [1, 1, 2, 2, 3, 3] },
+ { name: 'value', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
+ { name: 'region', type: FieldType.string, values: ['a', 'b', 'a', 'b', 'a', 'b'] },
+ ],
+ }),
+ ];
+
+ const config: PrepareTimeSeriesOptions = {
+ format: timeSeriesFormat.TimeSeriesMany,
+ };
+
+ const frames = prepareTimeSeriesTransformer.transformer(config)(source);
+ expect(frames).toEqual([
+ toEquableDataFrame({
+ name: 'long',
+ refId: 'X',
+ fields: [
+ { name: 'time', type: FieldType.time, values: [1, 2, 3] },
+ { name: 'value', labels: { region: 'a' }, type: FieldType.number, values: [10, 30, 50] },
+ ],
+ length: 3,
+ meta: {
+ type: DataFrameType.TimeSeriesMany,
+ },
+ }),
+ toEquableDataFrame({
+ name: 'long',
+ refId: 'X',
+ fields: [
+ { name: 'time', type: FieldType.time, values: [1, 2, 3] },
+ { name: 'value', labels: { region: 'b' }, type: FieldType.number, values: [20, 40, 60] },
+ ],
+ length: 3,
+ meta: {
+ type: DataFrameType.TimeSeriesMany,
+ },
+ }),
+ ]);
+ });
});
function toEquableDataFrame(source: any): DataFrame {
diff --git a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
index 42196c7adbc..4d237750620 100644
--- a/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
+++ b/public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts
@@ -7,7 +7,11 @@ import {
outerJoinDataFrames,
fieldMatchers,
FieldMatcherID,
+ Field,
+ MutableDataFrame,
+ ArrayVector,
} from '@grafana/data';
+import { Labels } from 'app/types/unified-alerting-dto';
import { map } from 'rxjs/operators';
/**
@@ -22,7 +26,7 @@ import { map } from 'rxjs/operators';
export enum timeSeriesFormat {
TimeSeriesWide = 'wide', // [time,...values]
TimeSeriesMany = 'many', // All frames have [time,number]
- // TimeSeriesLong = 'long',
+ TimeSeriesLong = 'long',
}
export type PrepareTimeSeriesOptions = {
@@ -37,40 +41,248 @@ export function toTimeSeriesMany(data: DataFrame[]): DataFrame[] {
return data;
}
+ const result: DataFrame[] = [];
+ for (const frame of toTimeSeriesLong(data)) {
+ const timeField = frame.fields[0];
+ if (!timeField || timeField.type !== FieldType.time) {
+ continue;
+ }
+ const valueFields: Field[] = [];
+ const labelFields: Field[] = [];
+ for (const field of frame.fields) {
+ switch (field.type) {
+ case FieldType.number:
+ case FieldType.boolean:
+ valueFields.push(field);
+ break;
+ case FieldType.string:
+ labelFields.push(field);
+ break;
+ }
+ }
+
+ for (const field of valueFields) {
+ if (labelFields.length) {
+ // new frame for each label key
+ type frameBuilder = {
+ time: number[];
+ value: number[];
+ key: string;
+ labels: Labels;
+ };
+ const builders = new Map();
+ for (let i = 0; i < frame.length; i++) {
+ const time = timeField.values.get(i);
+ const value = field.values.get(i);
+ if (value === undefined || time == null) {
+ continue; // skip values left over from join
+ }
+
+ const key = labelFields.map((f) => f.values.get(i)).join('/');
+ let builder = builders.get(key);
+ if (!builder) {
+ builder = {
+ key,
+ time: [],
+ value: [],
+ labels: {},
+ };
+ for (const label of labelFields) {
+ builder.labels[label.name] = label.values.get(i);
+ }
+ builders.set(key, builder);
+ }
+ builder.time.push(time);
+ builder.value.push(value);
+ }
+
+ // Add a frame for each distinct value
+ for (const b of builders.values()) {
+ result.push({
+ name: frame.name,
+ refId: frame.refId,
+ meta: {
+ ...frame.meta,
+ type: DataFrameType.TimeSeriesMany,
+ },
+ fields: [
+ {
+ ...timeField,
+ values: new ArrayVector(b.time),
+ },
+ {
+ ...field,
+ values: new ArrayVector(b.value),
+ labels: b.labels,
+ },
+ ],
+ length: b.time.length,
+ });
+ }
+ } else {
+ result.push({
+ name: frame.name,
+ refId: frame.refId,
+ meta: {
+ ...frame.meta,
+ type: DataFrameType.TimeSeriesMany,
+ },
+ fields: [timeField, field],
+ length: frame.length,
+ });
+ }
+ }
+ }
+ return result;
+}
+
+export function toTimeSeriesLong(data: DataFrame[]): DataFrame[] {
+ if (!Array.isArray(data) || data.length === 0) {
+ return data;
+ }
+
const result: DataFrame[] = [];
for (const frame of data) {
- const timeField = frame.fields.find((field) => {
- return field.type === FieldType.time;
- });
+ let timeField: Field | undefined;
+ const uniqueValueNames: string[] = [];
+ const uniqueValueNamesToType: Record = {};
+ const uniqueLabelKeys: Record = {};
+ const labelKeyToWideIndices: Record = {};
+ const uniqueFactorNamesToWideIndex: Record = {};
+
+ for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
+ const field = frame.fields[fieldIndex];
+
+ switch (field.type) {
+ case FieldType.string:
+ case FieldType.boolean:
+ if (field.name in uniqueFactorNamesToWideIndex) {
+ // TODO error?
+ } else {
+ uniqueFactorNamesToWideIndex[field.name] = fieldIndex;
+ uniqueLabelKeys[field.name] = true;
+ }
+ break;
+ case FieldType.time:
+ if (!timeField) {
+ timeField = field;
+ break;
+ }
+ default:
+ if (field.name in uniqueValueNamesToType) {
+ const type = uniqueValueNamesToType[field.name];
+
+ if (field.type !== type) {
+ // TODO error?
+ continue;
+ }
+ } else {
+ uniqueValueNamesToType[field.name] = field.type;
+ uniqueValueNames.push(field.name);
+ }
+
+ const tKey = JSON.stringify(field.labels);
+ const wideIndices = labelKeyToWideIndices[tKey];
+
+ if (wideIndices !== undefined) {
+ wideIndices.push(fieldIndex);
+ } else {
+ labelKeyToWideIndices[tKey] = [fieldIndex];
+ }
+
+ if (field.labels != null) {
+ for (const labelKey in field.labels) {
+ uniqueLabelKeys[labelKey] = true;
+ }
+ }
+ }
+ }
if (!timeField) {
continue;
}
- for (const field of frame.fields) {
- if (field.type !== FieldType.number) {
- continue;
- }
+ type TimeWideRowIndex = {
+ time: any;
+ wideRowIndex: number;
+ };
+ const sortedTimeRowIndices: TimeWideRowIndex[] = [];
+ const sortedUniqueLabelKeys: string[] = [];
+ const uniqueFactorNames: string[] = [];
+ const uniqueFactorNamesWithWideIndices: string[] = [];
- result.push({
- name: frame.name,
- refId: frame.refId,
- meta: {
- ...frame.meta,
- type: DataFrameType.TimeSeriesMany,
- },
- fields: [timeField, field],
- length: frame.length,
- });
+ for (let wideRowIndex = 0; wideRowIndex < frame.length; wideRowIndex++) {
+ sortedTimeRowIndices.push({ time: timeField.values.get(wideRowIndex), wideRowIndex: wideRowIndex });
}
+
+ for (const labelKeys in labelKeyToWideIndices) {
+ sortedUniqueLabelKeys.push(labelKeys);
+ }
+ for (const labelKey in uniqueLabelKeys) {
+ uniqueFactorNames.push(labelKey);
+ }
+ for (const name in uniqueFactorNamesToWideIndex) {
+ uniqueFactorNamesWithWideIndices.push(name);
+ }
+
+ sortedTimeRowIndices.sort((a, b) => a.time - b.time);
+ sortedUniqueLabelKeys.sort();
+ uniqueFactorNames.sort();
+ uniqueValueNames.sort();
+
+ const longFrame = new MutableDataFrame({
+ ...frame,
+ meta: { ...frame.meta, type: DataFrameType.TimeSeriesLong },
+ fields: [{ name: timeField.name, type: timeField.type }],
+ });
+
+ for (const name of uniqueValueNames) {
+ longFrame.addField({ name: name, type: uniqueValueNamesToType[name] });
+ }
+
+ for (const name of uniqueFactorNames) {
+ longFrame.addField({ name: name, type: FieldType.string });
+ }
+
+ for (const timeWideRowIndex of sortedTimeRowIndices) {
+ const { time, wideRowIndex } = timeWideRowIndex;
+
+ for (const labelKeys of sortedUniqueLabelKeys) {
+ const rowValues: Record = {};
+
+ for (const name of uniqueFactorNamesWithWideIndices) {
+ rowValues[name] = frame.fields[uniqueFactorNamesToWideIndex[name]].values.get(wideRowIndex);
+ }
+
+ let index = 0;
+
+ for (const wideFieldIndex of labelKeyToWideIndices[labelKeys]) {
+ const wideField = frame.fields[wideFieldIndex];
+
+ if (index++ === 0 && wideField.labels != null) {
+ for (const labelKey in wideField.labels) {
+ rowValues[labelKey] = wideField.labels[labelKey];
+ }
+ }
+
+ rowValues[wideField.name] = wideField.values.get(wideRowIndex);
+ }
+
+ rowValues[timeField.name] = time;
+ longFrame.add(rowValues);
+ }
+ }
+
+ result.push(longFrame);
}
+
return result;
}
export const prepareTimeSeriesTransformer: SynchronousDataTransformerInfo = {
id: DataTransformerID.prepareTimeSeries,
name: 'Prepare time series',
- description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatability for panels not supporting the new wide format.`,
+ description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatibility for panels not supporting the new wide format.`,
defaultOptions: {},
operator: (options) => (source) =>
@@ -80,6 +292,8 @@ export const prepareTimeSeriesTransformer: SynchronousDataTransformerInfo {
diff --git a/public/app/core/components/editors/DashboardPicker.tsx b/public/app/core/components/editors/DashboardPicker.tsx
new file mode 100644
index 00000000000..f806d44b300
--- /dev/null
+++ b/public/app/core/components/editors/DashboardPicker.tsx
@@ -0,0 +1,63 @@
+import React, { FC, useCallback, useState } from 'react';
+import debounce from 'debounce-promise';
+import { SelectableValue, StandardEditorProps } from '@grafana/data';
+import { DashboardSearchHit } from 'app/features/search/types';
+import { backendSrv } from 'app/core/services/backend_srv';
+import { AsyncSelect } from '@grafana/ui';
+import { useAsync } from 'react-use';
+
+export interface DashboardPickerOptions {
+ placeholder?: string;
+ isClearable?: boolean;
+}
+
+const getDashboards = (query = '') => {
+ return backendSrv.search({ type: 'dash-db', query, limit: 100 }).then((result: DashboardSearchHit[]) => {
+ return result.map((item: DashboardSearchHit) => ({
+ value: item.uid,
+ label: `${item?.folderTitle ?? 'General'}/${item.title}`,
+ }));
+ });
+};
+
+/** This will return the item UID */
+export const DashboardPicker: FC> = ({ value, onChange, item }) => {
+ const [current, setCurrent] = useState>();
+
+ // This is required because the async select does not match the raw uid value
+ // We can not use a simple Select because the dashboard search should not return *everything*
+ useAsync(async () => {
+ if (!value) {
+ setCurrent(undefined);
+ return;
+ }
+ const res = await backendSrv.getDashboardByUid(value);
+ setCurrent({
+ value: res.dashboard.uid,
+ label: `${res.meta?.folderTitle ?? 'General'}/${res.dashboard.title}`,
+ });
+ return undefined;
+ }, [value]);
+
+ const onPicked = useCallback(
+ (sel: SelectableValue) => {
+ onChange(sel?.value);
+ },
+ [onChange]
+ );
+ const debouncedSearch = debounce(getDashboards, 300);
+ const { placeholder, isClearable } = item?.settings ?? {};
+
+ return (
+
+ );
+};
diff --git a/public/app/core/components/Select/DashboardPicker.tsx b/public/app/core/components/editors/DashboardPickerByID.tsx
similarity index 69%
rename from public/app/core/components/Select/DashboardPicker.tsx
rename to public/app/core/components/editors/DashboardPickerByID.tsx
index 0803e825c70..33084541da9 100644
--- a/public/app/core/components/Select/DashboardPicker.tsx
+++ b/public/app/core/components/editors/DashboardPickerByID.tsx
@@ -5,14 +5,19 @@ import { AsyncSelect } from '@grafana/ui';
import { backendSrv } from 'app/core/services/backend_srv';
import { DashboardSearchHit } from 'app/features/search/types';
-export interface DashboardPickerItem extends Pick {
+/**
+ * @deprecated prefer using dashboard uid rather than id
+ */
+export interface DashboardPickerItem extends SelectableValue {
+ id: number;
+ uid: string;
value: number;
label: string;
}
-export interface Props {
+interface Props {
onChange: (dashboard: DashboardPickerItem) => void;
- value?: SelectableValue;
+ value?: DashboardPickerItem;
width?: number;
isClearable?: boolean;
invalid?: boolean;
@@ -20,7 +25,7 @@ export interface Props {
}
const getDashboards = (query = '') => {
- return backendSrv.search({ type: 'dash-db', query }).then((result: DashboardSearchHit[]) => {
+ return backendSrv.search({ type: 'dash-db', query, limit: 100 }).then((result: DashboardSearchHit[]) => {
return result.map((item: DashboardSearchHit) => ({
id: item.id,
uid: item.uid,
@@ -30,7 +35,10 @@ const getDashboards = (query = '') => {
});
};
-export const DashboardPicker: FC = ({ onChange, value, width, isClearable = false, invalid, disabled }) => {
+/**
+ * @deprecated prefer using dashboard uid rather than id
+ */
+export const DashboardPickerByID: FC = ({ onChange, value, width, isClearable = false, invalid, disabled }) => {
const debouncedSearch = debounce(getDashboards, 300);
return (
diff --git a/public/app/core/components/editors/registry.tsx b/public/app/core/components/editors/registry.tsx
new file mode 100644
index 00000000000..8c51fb4438c
--- /dev/null
+++ b/public/app/core/components/editors/registry.tsx
@@ -0,0 +1,16 @@
+import { DashboardPicker, DashboardPickerOptions } from './DashboardPicker';
+import { getStandardOptionEditors } from '@grafana/ui';
+import { StandardEditorsRegistryItem } from '@grafana/data';
+
+/**
+ * Returns collection of standard option editors definitions
+ */
+export const getAllOptionEditors = () => {
+ const dashboardPicker: StandardEditorsRegistryItem = {
+ id: 'dashboard-uid',
+ name: 'Dashboard',
+ description: 'Select dashboard',
+ editor: DashboardPicker as any,
+ };
+ return [...getStandardOptionEditors(), dashboardPicker];
+};
diff --git a/public/app/core/components/sidemenu/BottomSection.test.tsx b/public/app/core/components/sidemenu/BottomSection.test.tsx
index e0dca14ef2c..7cb4ac70e8c 100644
--- a/public/app/core/components/sidemenu/BottomSection.test.tsx
+++ b/public/app/core/components/sidemenu/BottomSection.test.tsx
@@ -1,11 +1,17 @@
import React from 'react';
import { render, screen } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
+import { BrowserRouter } from 'react-router-dom';
import { ShowModalReactEvent } from '../../../types/events';
import { HelpModal } from '../help/HelpModal';
import appEvents from '../../app_events';
import BottomSection from './BottomSection';
+jest.mock('./utils', () => ({
+ getForcedLoginUrl: () => '/mockForcedLoginUrl',
+ isLinkActive: () => false,
+ isSearchActive: () => false,
+}));
jest.mock('../../app_events', () => ({
publish: jest.fn(),
}));
@@ -44,13 +50,23 @@ jest.mock('app/core/services/context_srv', () => ({
describe('BottomSection', () => {
it('should render the correct children', () => {
- render();
+ render(
+
+
+
+ );
expect(screen.getByTestId('bottom-section-items').children.length).toBe(3);
});
it('creates the correct children for the help link', () => {
- render();
+ render(
+
+
+
+
+
+ );
const documentation = screen.getByRole('link', { name: 'Documentation' });
const support = screen.getByRole('link', { name: 'Support' });
@@ -63,7 +79,11 @@ describe('BottomSection', () => {
});
it('clicking the keyboard shortcuts button shows the modal', () => {
- render();
+ render(
+
+
+
+ );
const keyboardShortcuts = screen.getByText('Keyboard shortcuts');
expect(keyboardShortcuts).toBeInTheDocument();
@@ -73,7 +93,11 @@ describe('BottomSection', () => {
});
it('shows the current organization and organization switcher if showOrgSwitcher is true', () => {
- render();
+ render(
+
+
+
+ );
const currentOrg = screen.getByText(new RegExp('Grafana', 'i'));
const orgSwitcher = screen.getByText('Switch organization');
diff --git a/public/app/core/components/sidemenu/BottomSection.tsx b/public/app/core/components/sidemenu/BottomSection.tsx
index 5677d5d670b..f8fa6bbd879 100644
--- a/public/app/core/components/sidemenu/BottomSection.tsx
+++ b/public/app/core/components/sidemenu/BottomSection.tsx
@@ -1,21 +1,28 @@
import React, { useState } from 'react';
+import { useLocation } from 'react-router-dom';
import { cloneDeep } from 'lodash';
-import { NavModelItem } from '@grafana/data';
-import { Icon, IconName } from '@grafana/ui';
-import appEvents from '../../app_events';
-import { SignIn } from './SignIn';
-import SideMenuItem from './SideMenuItem';
-import { ShowModalReactEvent } from '../../../types/events';
+import { css } from '@emotion/css';
+import { GrafanaTheme2, NavModelItem } from '@grafana/data';
+import { Icon, IconName, styleMixins, useTheme2 } from '@grafana/ui';
import { contextSrv } from 'app/core/services/context_srv';
+import appEvents from '../../app_events';
+import { ShowModalReactEvent } from '../../../types/events';
+import config from '../../config';
import { OrgSwitcher } from '../OrgSwitcher';
import { getFooterLinks } from '../Footer/Footer';
import { HelpModal } from '../help/HelpModal';
-import config from '../../config';
+import SideMenuItem from './SideMenuItem';
+import { getForcedLoginUrl, isLinkActive, isSearchActive } from './utils';
export default function BottomSection() {
+ const theme = useTheme2();
+ const styles = getStyles(theme);
const navTree: NavModelItem[] = cloneDeep(config.bootData.navTree);
const bottomNav = navTree.filter((item) => item.hideFromMenu);
const isSignedIn = contextSrv.isSignedIn;
+ const location = useLocation();
+ const activeItemId = bottomNav.find((item) => isLinkActive(location.pathname, item))?.id;
+ const forcedLoginUrl = getForcedLoginUrl(location.pathname + location.search);
const user = contextSrv.user;
const [showSwitcherModal, setShowSwitcherModal] = useState(false);
@@ -36,8 +43,12 @@ export default function BottomSection() {
}
return (
-
- {!isSignedIn &&
}
+
+ {!isSignedIn && (
+
+
+
+ )}
{bottomNav.map((link, index) => {
let menuItems = link.children || [];
@@ -66,6 +77,7 @@ export default function BottomSection() {
return (
);
}
+
+const getStyles = (theme: GrafanaTheme2) => ({
+ container: css`
+ display: none;
+
+ @media ${styleMixins.mediaUp(`${theme.breakpoints.values.md}px`)} {
+ display: block;
+ margin-bottom: ${theme.spacing(2)};
+ }
+
+ .sidemenu-open--xs & {
+ display: block;
+ }
+ `,
+});
diff --git a/public/app/core/components/sidemenu/DropDownChild.tsx b/public/app/core/components/sidemenu/DropDownChild.tsx
index c8ec712c198..b69f4b233fe 100644
--- a/public/app/core/components/sidemenu/DropDownChild.tsx
+++ b/public/app/core/components/sidemenu/DropDownChild.tsx
@@ -1,5 +1,6 @@
import React from 'react';
import { css } from '@emotion/css';
+import { GrafanaTheme2 } from '@grafana/data';
import { Icon, IconName, Link, useTheme2 } from '@grafana/ui';
export interface Props {
@@ -13,35 +14,28 @@ export interface Props {
const DropDownChild = ({ isDivider = false, icon, onClick, target, text, url }: Props) => {
const theme = useTheme2();
- const iconClassName = css`
- margin-right: ${theme.spacing(1)};
- `;
- const resetButtonStyles = css`
- background-color: transparent;
- border: none;
- width: 100%;
- `;
+ const styles = getStyles(theme);
const linkContent = (
<>
- {icon && }
+ {icon && }
{text}
);
let element = (
-