diff --git a/.drone.star b/.drone.star index d1a09320e03..bac6e561223 100644 --- a/.drone.star +++ b/.drone.star @@ -2,8 +2,9 @@ load('scripts/pr.star', 'pr_pipelines') load('scripts/main.star', 'main_pipelines') load('scripts/release.star', 'release_pipelines', 'test_release_pipelines') load('scripts/version.star', 'version_branch_pipelines') +load('scripts/vault.star', 'secrets') def main(ctx): edition = 'oss' return pr_pipelines(edition=edition) + main_pipelines(edition=edition) + release_pipelines() + \ - test_release_pipelines() + version_branch_pipelines() + test_release_pipelines() + version_branch_pipelines() + secrets() diff --git a/.drone.yml b/.drone.yml index f0705b1ba9b..506649ea05e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1476,6 +1476,9 @@ services: - name: memcached image: memcached:1.6.9-alpine +image_pull_secrets: +- dockerconfigjson + trigger: ref: - refs/tags/v* @@ -1543,6 +1546,9 @@ steps: depends_on: - initialize +image_pull_secrets: +- dockerconfigjson + trigger: ref: - refs/tags/v* @@ -2408,6 +2414,9 @@ services: - name: memcached image: memcached:1.6.9-alpine +image_pull_secrets: +- dockerconfigjson + trigger: event: - custom @@ -2475,6 +2484,9 @@ steps: depends_on: - initialize +image_pull_secrets: +- dockerconfigjson + trigger: event: - custom @@ -3314,6 +3326,9 @@ services: - name: memcached image: memcached:1.6.9-alpine +image_pull_secrets: +- dockerconfigjson + trigger: ref: - refs/heads/v* @@ -3377,6 +3392,9 @@ steps: depends_on: - initialize +image_pull_secrets: +- dockerconfigjson + trigger: ref: - refs/heads/v* @@ -3414,4 +3432,20 @@ depends_on: - enterprise-build-release-branch - enterprise-windows-release-branch +--- +kind: secret +name: dockerconfigjson + +get: + path: secret/data/common/gcr + name: .dockerconfigjson + +--- +kind: secret +name: github_token + +get: + path: infra/data/ci/github/grafanabot + name: pat + ... diff --git a/packages/grafana-data/src/dataframe/index.ts b/packages/grafana-data/src/dataframe/index.ts index 95b7cf951c4..a16bc33e3bc 100644 --- a/packages/grafana-data/src/dataframe/index.ts +++ b/packages/grafana-data/src/dataframe/index.ts @@ -8,3 +8,4 @@ export * from './ArrayDataFrame'; export * from './DataFrameJSON'; export * from './StreamingDataFrame'; export * from './frameComparisons'; +export { anySeriesWithTimeField } from './utils'; diff --git a/packages/grafana-data/src/dataframe/utils.test.ts b/packages/grafana-data/src/dataframe/utils.test.ts new file mode 100644 index 00000000000..943a1d900bf --- /dev/null +++ b/packages/grafana-data/src/dataframe/utils.test.ts @@ -0,0 +1,78 @@ +import { toDataFrame } from './processDataFrame'; +import { FieldType } from '../types'; +import { anySeriesWithTimeField } from './utils'; + +describe('anySeriesWithTimeField', () => { + describe('single frame', () => { + test('without time field', () => { + const frameA = toDataFrame({ + fields: [ + { name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] }, + { name: 'value', type: FieldType.number, values: [1, 2, 3] }, + ], + }); + expect(anySeriesWithTimeField([frameA])).toBeFalsy(); + }); + + test('with time field', () => { + const frameA = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] }, + { name: 'value', type: FieldType.number, values: [1, 2, 3] }, + ], + }); + expect(anySeriesWithTimeField([frameA])).toBeTruthy(); + }); + }); + + describe('multiple frames', () => { + test('without time field', () => { + const frameA = toDataFrame({ + fields: [ + { name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] }, + { name: 'value', type: FieldType.number, values: [1, 2, 3] }, + ], + }); + const frameB = toDataFrame({ + fields: [{ name: 'value', type: FieldType.number, values: [1, 2, 3] }], + }); + expect(anySeriesWithTimeField([frameA, frameB])).toBeFalsy(); + }); + + test('with time field in any frame', () => { + const frameA = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] }, + { name: 'value', type: FieldType.number, values: [1, 2, 3] }, + ], + }); + const frameB = toDataFrame({ + fields: [{ name: 'value', type: FieldType.number, values: [1, 2, 3] }], + }); + const frameC = toDataFrame({ + fields: [{ name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] }], + }); + + expect(anySeriesWithTimeField([frameA, frameB, frameC])).toBeTruthy(); + }); + + test('with time field in a all frames', () => { + const frameA = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'value', type: FieldType.number, values: [1, 2, 3] }, + ], + }); + const frameB = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] }, + { name: 'value', type: FieldType.number, values: [1, 2, 3] }, + ], + }); + expect(anySeriesWithTimeField([frameA, frameB])).toBeTruthy(); + }); + }); +}); diff --git a/packages/grafana-data/src/dataframe/utils.ts b/packages/grafana-data/src/dataframe/utils.ts index 5e9f1300791..4044393d839 100644 --- a/packages/grafana-data/src/dataframe/utils.ts +++ b/packages/grafana-data/src/dataframe/utils.ts @@ -1,12 +1,27 @@ import { DataFrame, FieldType } from '../types/dataFrame'; +import { getTimeField } from './processDataFrame'; -export const isTimeSerie = (frame: DataFrame): boolean => { +export function isTimeSerie(frame: DataFrame) { if (frame.fields.length > 2) { return false; } - return !!frame.fields.find((field) => field.type === FieldType.time); -}; + return Boolean(frame.fields.find((field) => field.type === FieldType.time)); +} -export const isTimeSeries = (data: DataFrame[]): boolean => { +export function isTimeSeries(data: DataFrame[]) { return !data.find((frame) => !isTimeSerie(frame)); -}; +} + +/** + * Indicates if there is any time field in the array of data frames + * @param data + */ +export function anySeriesWithTimeField(data: DataFrame[]) { + for (let i = 0; i < data.length; i++) { + const timeField = getTimeField(data[i]); + if (timeField.timeField !== undefined && timeField.timeIndex !== undefined) { + return true; + } + } + return false; +} diff --git a/packages/grafana-data/src/types/logs.ts b/packages/grafana-data/src/types/logs.ts index 348dd316587..cbc987b798f 100644 --- a/packages/grafana-data/src/types/logs.ts +++ b/packages/grafana-data/src/types/logs.ts @@ -2,6 +2,7 @@ import { Labels } from './data'; import { GraphSeriesXY } from './graph'; import { DataFrame } from './dataFrame'; import { AbsoluteTimeRange } from './time'; +import { DataQuery } from './datasource'; /** * Mapping of log level abbreviation to canonical log level. @@ -85,6 +86,7 @@ export interface LogsModel { rows: LogRowModel[]; series?: GraphSeriesXY[]; visibleRange?: AbsoluteTimeRange; + queries?: DataQuery[]; } export interface LogSearchMatch { diff --git a/packages/grafana-data/src/utils/url.test.ts b/packages/grafana-data/src/utils/url.test.ts index c1a8bfd07e4..27f93d2ff78 100644 --- a/packages/grafana-data/src/utils/url.test.ts +++ b/packages/grafana-data/src/utils/url.test.ts @@ -37,16 +37,21 @@ describe('parseKeyValue', () => { it('should parse numeric params', () => { const obj = urlUtil.parseKeyValue('num1=12&num2=12.2'); - expect(obj).toEqual({ num1: 12, num2: 12.2 }); + expect(obj).toEqual({ num1: '12', num2: '12.2' }); }); - it('should not parse empty strinhg as number', () => { + it('should not parse empty string as number', () => { const obj = urlUtil.parseKeyValue('num1=&num2=12.2'); - expect(obj).toEqual({ num1: '', num2: 12.2 }); + expect(obj).toEqual({ num1: '', num2: '12.2' }); }); it('should parse boolean params', () => { const obj = urlUtil.parseKeyValue('bool1&bool2=true&bool3=false'); expect(obj).toEqual({ bool1: true, bool2: true, bool3: false }); }); + + it('should parse number like params as strings', () => { + const obj = urlUtil.parseKeyValue('custom=&custom1=001&custom2=002&custom3'); + expect(obj).toEqual({ custom: '', custom1: '001', custom2: '002', custom3: true }); + }); }); diff --git a/packages/grafana-data/src/utils/url.ts b/packages/grafana-data/src/utils/url.ts index c733cf137ea..8d6de90e35e 100644 --- a/packages/grafana-data/src/utils/url.ts +++ b/packages/grafana-data/src/utils/url.ts @@ -2,7 +2,6 @@ * @preserve jquery-param (c) 2015 KNOWLEDGECODE | MIT */ -import { toNumber } from 'lodash'; import { ExploreUrlState } from '../types/explore'; /** @@ -156,7 +155,7 @@ export function parseKeyValue(keyValue: string) { let parsedVal: any; if (typeof val === 'string' && val !== '') { - parsedVal = val === 'true' || val === 'false' ? val === 'true' : toNumber(val); + parsedVal = val === 'true' || val === 'false' ? val === 'true' : val; } else { parsedVal = val; } diff --git a/packages/grafana-runtime/src/services/LocationService.test.ts b/packages/grafana-runtime/src/services/LocationService.test.ts index efbb08c37fe..c01282b5ed5 100644 --- a/packages/grafana-runtime/src/services/LocationService.test.ts +++ b/packages/grafana-runtime/src/services/LocationService.test.ts @@ -7,7 +7,7 @@ describe('LocationService', () => { expect(locationService.getSearchObject()).toEqual({ query1: false, - query2: 123, + query2: '123', query3: 'text', }); }); diff --git a/packages/grafana-ui/src/components/SecretFormField/SecretFormField.story.internal.tsx b/packages/grafana-ui/src/components/SecretFormField/SecretFormField.story.internal.tsx index 8292975e160..0fa8337edb8 100644 --- a/packages/grafana-ui/src/components/SecretFormField/SecretFormField.story.internal.tsx +++ b/packages/grafana-ui/src/components/SecretFormField/SecretFormField.story.internal.tsx @@ -1,8 +1,8 @@ import React from 'react'; import { action } from '@storybook/addon-actions'; -import { boolean } from '@storybook/addon-knobs'; +import { Meta, Story } from '@storybook/react'; -import { SecretFormField } from './SecretFormField'; +import { SecretFormField, Props } from './SecretFormField'; import { withCenteredStory } from '../../utils/storybook/withCenteredStory'; import { UseState } from '../../utils/storybook/UseState'; @@ -10,31 +10,48 @@ export default { title: 'Forms/SecretFormField', component: SecretFormField, decorators: [withCenteredStory], -}; + parameters: { + controls: { + exclude: ['onReset'], + }, + knobs: { + disable: true, + }, + }, + argTypes: { + labelWidth: { control: { type: 'range', min: 0, max: 30 } }, + inputWidth: { control: { type: 'range', min: 0, max: 30 } }, + tooltip: { control: { type: 'text' } }, + }, +} as Meta; -const getSecretFormFieldKnobs = () => { - return { - isConfigured: boolean('Set configured state', false), - }; -}; - -export const basic = () => { - const knobs = getSecretFormFieldKnobs(); +export const Basic: Story = (args) => { return ( {(value, setValue) => ( setValue(e.currentTarget.value)} onReset={() => { action('Value was reset')(''); setValue(''); }} + inputWidth={args.inputWidth} + tooltip={args.tooltip} + placeholder={args.placeholder} /> )} ); }; +Basic.args = { + label: 'Secret field', + labelWidth: 10, + isConfigured: false, + inputWidth: 12, + tooltip: 'this is a tooltip', + placeholder: 'Password', +}; diff --git a/packages/grafana-ui/src/components/Select/Select.story.tsx b/packages/grafana-ui/src/components/Select/Select.story.tsx index af46b5d38e4..e6e12b4e4c2 100644 --- a/packages/grafana-ui/src/components/Select/Select.story.tsx +++ b/packages/grafana-ui/src/components/Select/Select.story.tsx @@ -3,10 +3,12 @@ import { withCenteredStory, withHorizontallyCenteredStory } from '../../utils/st import { SelectableValue } from '@grafana/data'; import { Icon, Select, AsyncSelect, MultiSelect, AsyncMultiSelect } from '@grafana/ui'; import { getAvailableIcons, IconName } from '../../types'; -import { select, boolean, number } from '@storybook/addon-knobs'; +import { SelectCommonProps } from './types'; +import { Meta, Story } from '@storybook/react'; import { kebabCase } from 'lodash'; import { generateOptions } from './mockOptions'; import mdx from './Select.mdx'; +import { auto } from '@popperjs/core'; export default { title: 'Forms/Select', @@ -17,10 +19,56 @@ export default { docs: { page: mdx, }, + knobs: { + disable: true, + }, + controls: { + exclude: [ + 'getOptionValue', + 'getOptionLabel', + 'formatCreateLabel', + 'filterOption', + 'className', + 'components', + 'defaultValue', + 'id', + 'inputId', + 'onBlur', + 'onChange', + 'onCloseMenu', + 'onCreateOption', + 'onInputChange', + 'onKeyDown', + 'onOpenMenu', + 'prefix', + 'renderControl', + 'options', + 'isOptionDisabled', + 'maxVisibleValues', + 'aria-label', + 'noOptionsMessage', + 'menuPosition', + 'value', + ], + }, }, -}; - -const BEHAVIOUR_GROUP = 'Behaviour props'; + args: { + width: 0, + disabled: false, + isLoading: false, + invalid: false, + icon: 'arrow-down', + }, + argTypes: { + width: { control: { type: 'range', min: 1, max: 100 } }, + icon: { + control: { + type: 'select', + options: getAvailableIcons(), + }, + }, + }, +} as Meta; const loadAsyncOptions = () => { return new Promise>>((resolve) => { @@ -30,63 +78,16 @@ const loadAsyncOptions = () => { }); }; -const getKnobs = () => { - const disabled = boolean('Disabled', false, BEHAVIOUR_GROUP); - const invalid = boolean('Invalid', false, BEHAVIOUR_GROUP); - const loading = boolean('Loading', false, BEHAVIOUR_GROUP); - const prefixSuffixOpts = { - None: null, - Text: '$', - ...getAvailableIcons().reduce>((prev, c) => { - return { - ...prev, - [`Icon: ${c}`]: `icon-${c}`, - }; - }, {}), - }; - const VISUAL_GROUP = 'Visual options'; - // --- - const prefix = select('Prefix', prefixSuffixOpts, null, VISUAL_GROUP); - const width = number('Width', 0, undefined, VISUAL_GROUP); - - let prefixEl: any = prefix; - if (prefix && prefix.match(/icon-/g)) { - prefixEl = ; - } - - return { - width, - disabled, - invalid, - loading, - prefixEl, - }; +const getPrefix = (prefix: string) => { + const prefixEl = ; + return prefixEl; }; -const getMultiSelectKnobs = () => { - const isClearable = boolean('Clearable', false, BEHAVIOUR_GROUP); - const closeMenuOnSelect = boolean('Close on Select', false, BEHAVIOUR_GROUP); - const maxVisibleValues = number('Max. visible values', 5, undefined, BEHAVIOUR_GROUP); +interface StoryProps extends Partial> { + icon: string; +} - return { - isClearable, - closeMenuOnSelect, - maxVisibleValues, - }; -}; - -const getDynamicProps = () => { - const knobs = getKnobs(); - return { - width: knobs.width, - disabled: knobs.disabled, - isLoading: knobs.loading, - invalid: knobs.invalid, - prefix: knobs.prefixEl, - }; -}; - -export const Basic = () => { +export const Basic: Story = (args) => { const [value, setValue] = useState>(); return ( @@ -97,16 +98,16 @@ export const Basic = () => { onChange={(v) => { setValue(v); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; - /** * Uses plain values instead of SelectableValue */ -export const BasicSelectPlainValue = () => { +export const BasicSelectPlainValue: Story = (args) => { const [value, setValue] = useState(); return ( <> @@ -116,16 +117,16 @@ export const BasicSelectPlainValue = () => { onChange={(v) => { setValue(v.value); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; - /** * Uses plain values instead of SelectableValue */ -export const SelectWithOptionDescriptions = () => { +export const SelectWithOptionDescriptions: Story = (args) => { // TODO this is not working with new Select const [value, setValue] = useState(); @@ -148,7 +149,8 @@ export const SelectWithOptionDescriptions = () => { onChange={(v) => { setValue(v.value); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); @@ -157,7 +159,7 @@ export const SelectWithOptionDescriptions = () => { /** * Uses plain values instead of SelectableValue */ -export const MultiPlainValue = () => { +export const MultiPlainValue: Story = (args) => { const [value, setValue] = useState(); return ( @@ -168,13 +170,14 @@ export const MultiPlainValue = () => { onChange={(v) => { setValue(v.map((v: any) => v.value)); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; -export const MultiSelectWithOptionGroups = () => { +export const MultiSelectWithOptionGroups: Story = (args) => { const [value, setValue] = useState(); return ( @@ -188,13 +191,14 @@ export const MultiSelectWithOptionGroups = () => { onChange={(v) => { setValue(v.map((v: any) => v.value)); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; -export const MultiSelectBasic = () => { +export const MultiSelectBasic: Story = (args) => { const [value, setValue] = useState>>([]); return ( @@ -205,14 +209,19 @@ export const MultiSelectBasic = () => { onChange={(v) => { setValue(v); }} - {...getDynamicProps()} - {...getMultiSelectKnobs()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; +MultiSelectBasic.args = { + isClearable: false, + closeMenuOnSelect: false, + maxVisibleValues: 5, +}; -export const MultiSelectAsync = () => { +export const MultiSelectAsync: Story = (args) => { const [value, setValue] = useState>>(); return ( @@ -223,13 +232,16 @@ export const MultiSelectAsync = () => { onChange={(v) => { setValue(v); }} - allowCustomValue - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; +MultiSelectAsync.args = { + allowCustomValue: false, +}; -export const BasicSelectAsync = () => { +export const BasicSelectAsync: Story = (args) => { const [value, setValue] = useState>(); return ( @@ -240,12 +252,13 @@ export const BasicSelectAsync = () => { onChange={(v) => { setValue(v); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; -export const AutoMenuPlacement = () => { +export const AutoMenuPlacement: Story = (args) => { const [value, setValue] = useState>(); return ( @@ -257,14 +270,18 @@ export const AutoMenuPlacement = () => { onChange={(v) => { setValue(v); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; +AutoMenuPlacement.args = { + menuPlacement: auto, +}; -export const CustomValueCreation = () => { +export const CustomValueCreation: Story = (args) => { const [value, setValue] = useState>(); const [customOptions, setCustomOptions] = useState>>([]); const options = generateOptions(); @@ -276,14 +293,18 @@ export const CustomValueCreation = () => { onChange={(v) => { setValue(v); }} - allowCustomValue + allowCustomValue={args.allowCustomValue} onCreateOption={(v) => { const customValue: SelectableValue = { value: kebabCase(v), label: v }; setCustomOptions([...customOptions, customValue]); setValue(customValue); }} - {...getDynamicProps()} + prefix={getPrefix(args.icon)} + {...args} /> ); }; +CustomValueCreation.args = { + allowCustomValue: true, +}; diff --git a/packages/grafana-ui/src/components/Slider/Slider.story.tsx b/packages/grafana-ui/src/components/Slider/Slider.story.tsx index 7b98c43bc8e..9a99cfd7d4b 100644 --- a/packages/grafana-ui/src/components/Slider/Slider.story.tsx +++ b/packages/grafana-ui/src/components/Slider/Slider.story.tsx @@ -1,30 +1,47 @@ import React from 'react'; import { Slider } from '@grafana/ui'; -import { select, number, boolean } from '@storybook/addon-knobs'; +import { SliderProps } from './types'; +import { Story, Meta } from '@storybook/react'; export default { title: 'Forms/Slider', component: Slider, -}; + parameters: { + controls: { + exclude: ['step', 'formatTooltipResult', 'onChange', 'onAfterChange', 'value', 'tooltipAlwaysVisible'], + }, + knobs: { + disabled: true, + }, + }, + argTypes: { + isStep: { name: 'Step' }, + orientation: { control: { type: 'select', options: ['horizontal', 'vertical'] } }, + }, +} as Meta; -const getKnobs = () => { - return { - min: number('min', 0), - max: number('max', 100), - step: boolean('enable step', false), - orientation: select('orientation', ['horizontal', 'vertical'], 'horizontal'), - reverse: boolean('reverse', false), - }; -}; +interface StoryProps extends Partial { + isStep: boolean; +} -const SliderWrapper = () => { - const { min, max, orientation, reverse, step } = getKnobs(); - const stepValue = step ? 10 : undefined; +export const Basic: Story = (args) => { return (
- +
); }; - -export const basic = () => ; +Basic.args = { + min: 0, + max: 100, + value: 10, + isStep: false, + orientation: 'horizontal', + reverse: false, +}; diff --git a/packages/grafana-ui/src/components/TagsInput/TagsInput.tsx b/packages/grafana-ui/src/components/TagsInput/TagsInput.tsx index 70a2727a79d..095e1e739df 100644 --- a/packages/grafana-ui/src/components/TagsInput/TagsInput.tsx +++ b/packages/grafana-ui/src/components/TagsInput/TagsInput.tsx @@ -11,9 +11,16 @@ export interface Props { tags?: string[]; onChange: (tags: string[]) => void; width?: number; + className?: string; } -export const TagsInput: FC = ({ placeholder = 'New tag (enter key to add)', tags = [], onChange, width }) => { +export const TagsInput: FC = ({ + placeholder = 'New tag (enter key to add)', + tags = [], + onChange, + width, + className, +}) => { const [newTagName, setNewName] = useState(''); const styles = useStyles(getStyles); const theme = useTheme2(); @@ -28,7 +35,9 @@ export const TagsInput: FC = ({ placeholder = 'New tag (enter key to add) const onAdd = (event: React.MouseEvent) => { event.preventDefault(); - onChange(tags.concat(newTagName)); + if (!tags.includes(newTagName)) { + onChange(tags.concat(newTagName)); + } setNewName(''); }; @@ -41,7 +50,7 @@ export const TagsInput: FC = ({ placeholder = 'New tag (enter key to add) }; return ( -
+
{tags?.map((tag: string, index: number) => { return ; diff --git a/pkg/api/api.go b/pkg/api/api.go index 8524fb49cde..1b39d9f0fa7 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -203,6 +203,7 @@ func (hs *HTTPServer) registerRoutes() { orgRoute.Put("/", reqOrgAdmin, bind(dtos.UpdateOrgForm{}), routing.Wrap(UpdateOrgCurrent)) orgRoute.Put("/address", reqOrgAdmin, bind(dtos.UpdateOrgAddressForm{}), routing.Wrap(UpdateOrgAddressCurrent)) orgRoute.Get("/users", authorize(reqOrgAdmin, accesscontrol.ActionOrgUsersRead, accesscontrol.ScopeUsersAll), routing.Wrap(hs.GetOrgUsersForCurrentOrg)) + orgRoute.Get("/users/search", authorize(reqOrgAdmin, accesscontrol.ActionOrgUsersRead, accesscontrol.ScopeUsersAll), routing.Wrap(hs.SearchOrgUsersWithPaging)) orgRoute.Post("/users", authorize(reqOrgAdmin, accesscontrol.ActionOrgUsersAdd, accesscontrol.ScopeUsersAll), quota("user"), bind(models.AddOrgUserCommand{}), routing.Wrap(AddOrgUserToCurrentOrg)) orgRoute.Patch("/users/:userId", authorize(reqOrgAdmin, accesscontrol.ActionOrgUsersRoleUpdate, usersScope), bind(models.UpdateOrgUserCommand{}), routing.Wrap(UpdateOrgUserForCurrentOrg)) orgRoute.Delete("/users/:userId", authorize(reqOrgAdmin, accesscontrol.ActionOrgUsersRemove, usersScope), routing.Wrap(RemoveOrgUserForCurrentOrg)) diff --git a/pkg/api/org_users.go b/pkg/api/org_users.go index 7c2dc3e6457..61f8f19c1e9 100644 --- a/pkg/api/org_users.go +++ b/pkg/api/org_users.go @@ -157,6 +157,47 @@ func (hs *HTTPServer) getOrgUsersHelper(query *models.GetOrgUsersQuery, signedIn return filteredUsers, nil } +// SearchOrgUsersWithPaging is an HTTP handler to search for org users with paging. +// GET /api/org/users/search +func (hs *HTTPServer) SearchOrgUsersWithPaging(c *models.ReqContext) response.Response { + perPage := c.QueryInt("perpage") + if perPage <= 0 { + perPage = 1000 + } + page := c.QueryInt("page") + + if page < 1 { + page = 1 + } + + query := &models.SearchOrgUsersQuery{ + OrgID: c.OrgId, + Query: c.Query("query"), + Limit: perPage, + Page: page, + } + + if err := hs.SQLStore.SearchOrgUsers(query); err != nil { + return response.Error(500, "Failed to get users for current organization", err) + } + + filteredUsers := make([]*models.OrgUserDTO, 0, len(query.Result.OrgUsers)) + for _, user := range query.Result.OrgUsers { + if dtos.IsHiddenUser(user.Login, c.SignedInUser, hs.Cfg) { + continue + } + user.AvatarUrl = dtos.GetGravatarUrl(user.Email) + + filteredUsers = append(filteredUsers, user) + } + + query.Result.OrgUsers = filteredUsers + query.Result.Page = page + query.Result.PerPage = perPage + + return response.JSON(200, query.Result) +} + // PATCH /api/org/users/:userId func UpdateOrgUserForCurrentOrg(c *models.ReqContext, cmd models.UpdateOrgUserCommand) response.Response { cmd.OrgId = c.OrgId @@ -175,7 +216,6 @@ func updateOrgUserHelper(cmd models.UpdateOrgUserCommand) response.Response { if !cmd.Role.IsValid() { return response.Error(400, "Invalid role specified", nil) } - if err := bus.Dispatch(&cmd); err != nil { if errors.Is(err, models.ErrLastOrgAdmin) { return response.Error(400, "Cannot change role so that there is no organization admin left", nil) diff --git a/pkg/api/org_users_test.go b/pkg/api/org_users_test.go index 34445e128b9..8b71a024212 100644 --- a/pkg/api/org_users_test.go +++ b/pkg/api/org_users_test.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "net/http" "testing" @@ -8,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,10 +26,24 @@ func setUpGetOrgUsersHandler() { }) } +func setUpGetOrgUsersDB(t *testing.T, sqlStore *sqlstore.SQLStore) { + setting.AutoAssignOrg = true + setting.AutoAssignOrgId = 1 + + _, err := sqlStore.CreateUser(context.Background(), models.CreateUserCommand{Email: "testUser@grafana.com", Login: "testUserLogin"}) + require.NoError(t, err) + _, err = sqlStore.CreateUser(context.Background(), models.CreateUserCommand{Email: "user1@grafana.com", Login: "user1"}) + require.NoError(t, err) + _, err = sqlStore.CreateUser(context.Background(), models.CreateUserCommand{Email: "user2@grafana.com", Login: "user2"}) + require.NoError(t, err) +} + func TestOrgUsersAPIEndpoint_userLoggedIn(t *testing.T) { settings := setting.NewCfg() hs := &HTTPServer{Cfg: settings} + sqlStore := sqlstore.InitTestDB(t) + loggedInUserScenario(t, "When calling GET on", "api/org/users", func(sc *scenarioContext) { setUpGetOrgUsersHandler() @@ -42,6 +58,42 @@ func TestOrgUsersAPIEndpoint_userLoggedIn(t *testing.T) { assert.Len(t, resp, 3) }) + loggedInUserScenario(t, "When calling GET on", "api/org/users/search", func(sc *scenarioContext) { + setUpGetOrgUsersDB(t, sqlStore) + + sc.handlerFunc = hs.SearchOrgUsersWithPaging + sc.fakeReqWithParams("GET", sc.url, map[string]string{}).exec() + + require.Equal(t, http.StatusOK, sc.resp.Code) + + var resp models.SearchOrgUsersQueryResult + err := json.Unmarshal(sc.resp.Body.Bytes(), &resp) + require.NoError(t, err) + + assert.Len(t, resp.OrgUsers, 3) + assert.Equal(t, int64(3), resp.TotalCount) + assert.Equal(t, 1000, resp.PerPage) + assert.Equal(t, 1, resp.Page) + }) + + loggedInUserScenario(t, "When calling GET with page and limit query parameters on", "api/org/users/search", func(sc *scenarioContext) { + setUpGetOrgUsersDB(t, sqlStore) + + sc.handlerFunc = hs.SearchOrgUsersWithPaging + sc.fakeReqWithParams("GET", sc.url, map[string]string{"perpage": "2", "page": "2"}).exec() + + require.Equal(t, http.StatusOK, sc.resp.Code) + + var resp models.SearchOrgUsersQueryResult + err := json.Unmarshal(sc.resp.Body.Bytes(), &resp) + require.NoError(t, err) + + assert.Len(t, resp.OrgUsers, 1) + assert.Equal(t, int64(3), resp.TotalCount) + assert.Equal(t, 2, resp.PerPage) + assert.Equal(t, 2, resp.Page) + }) + loggedInUserScenario(t, "When calling GET as an editor with no team / folder permissions on", "api/org/users/lookup", func(sc *scenarioContext) { setUpGetOrgUsersHandler() diff --git a/pkg/models/org_user.go b/pkg/models/org_user.go index e3823f880a1..c432746852e 100644 --- a/pkg/models/org_user.go +++ b/pkg/models/org_user.go @@ -114,6 +114,22 @@ type GetOrgUsersQuery struct { Result []*OrgUserDTO } +type SearchOrgUsersQuery struct { + OrgID int64 + Query string + Page int + Limit int + + Result SearchOrgUsersQueryResult +} + +type SearchOrgUsersQueryResult struct { + TotalCount int64 `json:"totalCount"` + OrgUsers []*OrgUserDTO `json:"OrgUsers"` + Page int `json:"page"` + PerPage int `json:"perPage"` +} + // ---------------------- // Projections and DTOs diff --git a/pkg/schema/load/applydefault_test.go b/pkg/schema/load/applydefault_test.go deleted file mode 100644 index e429843b4a6..00000000000 --- a/pkg/schema/load/applydefault_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package load - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "path/filepath" - "testing" - - "cuelang.org/go/cue" - "github.com/google/go-cmp/cmp" - "github.com/grafana/grafana/pkg/schema" - "golang.org/x/tools/txtar" -) - -var CasesDir = filepath.Join("testdata", "artifacts", "dashboards", "trimdefault") - -type Case struct { - Name string - - CUE string - Full string - Trimed string -} - -func TestGenerate(t *testing.T) { - cases, err := loadCases(CasesDir) - if err != nil { - t.Fatal(err) - } - - for _, c := range cases { - t.Run(c.Name+" apply default value", func(t *testing.T) { - var r cue.Runtime - scmInstance, err := r.Compile(c.Name+".cue", c.CUE) - if err != nil { - t.Fatal(err) - } - inputResource := schema.Resource{Value: c.Trimed} - scm := genericVersionedSchema{actual: scmInstance.Value()} - out, err := scm.ApplyDefaults(inputResource) - if err != nil { - t.Fatal(err) - } - b := []byte(out.Value.(string)) - - if s := cmp.Diff(string(b), c.Full); s != "" { - t.Fatal(s) - } - }) - } - - for _, c := range cases { - t.Run(c.Name+" trim default value", func(t *testing.T) { - var r cue.Runtime - scmInstance, err := r.Compile(c.Name+".cue", c.CUE) - if err != nil { - t.Fatal(err) - } - inputResource := schema.Resource{Value: c.Full} - scm := genericVersionedSchema{actual: scmInstance.Value()} - out, err := scm.TrimDefaults(inputResource) - if err != nil { - t.Fatal(err) - } - b := []byte(out.Value.(string)) - if s := cmp.Diff(string(b), c.Trimed); s != "" { - t.Fatal(s) - } - }) - } -} - -func loadCases(dir string) ([]Case, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - - var cases []Case - - for _, fi := range files { - file := filepath.Join(dir, fi.Name()) - a, err := txtar.ParseFile(file) - if err != nil { - return nil, err - } - - if len(a.Files) != 3 { - return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files)) - } - - fullBuffer := new(bytes.Buffer) - fullJson := a.Files[1].Data - if err := json.Compact(fullBuffer, fullJson); err != nil { - return nil, err - } - - trimBuffer := new(bytes.Buffer) - trimedJson := a.Files[2].Data - if err := json.Compact(trimBuffer, trimedJson); err != nil { - return nil, err - } - - cases = append(cases, Case{ - Name: fi.Name(), - CUE: string(a.Files[0].Data), - Full: fullBuffer.String(), - Trimed: trimBuffer.String(), - }) - } - - return cases, nil -} diff --git a/pkg/schema/load/dashboard.go b/pkg/schema/load/dashboard.go index 08569c3b596..5bc98e425e5 100644 --- a/pkg/schema/load/dashboard.go +++ b/pkg/schema/load/dashboard.go @@ -129,20 +129,6 @@ func (cds *compositeDashboardSchema) Validate(r schema.Resource) error { return cds.actual.Unify(rv.Value()).Validate(cue.Concrete(true)) } -// ApplyDefaults returns a new, concrete copy of the Resource with all paths -// that are 1) missing in the Resource AND 2) specified by the schema, -// filled with default values specified by the schema. -func (cds *compositeDashboardSchema) ApplyDefaults(_ schema.Resource) (schema.Resource, error) { - panic("not implemented") // TODO: Implement -} - -// TrimDefaults returns a new, concrete copy of the Resource where all paths -// in the where the values at those paths are the same as the default value -// given in the schema. -func (cds *compositeDashboardSchema) TrimDefaults(_ schema.Resource) (schema.Resource, error) { - panic("not implemented") // TODO: Implement -} - // CUE returns the cue.Value representing the actual schema. func (cds *compositeDashboardSchema) CUE() cue.Value { return cds.actual diff --git a/pkg/schema/load/generic.go b/pkg/schema/load/generic.go index 1cb0019141e..bedded41e48 100644 --- a/pkg/schema/load/generic.go +++ b/pkg/schema/load/generic.go @@ -1,13 +1,8 @@ package load import ( - "bytes" - "fmt" - "strings" - "cuelang.org/go/cue" "cuelang.org/go/cue/load" - cuejson "cuelang.org/go/pkg/encoding/json" "github.com/grafana/grafana/pkg/schema" ) @@ -108,142 +103,6 @@ func (gvs *genericVersionedSchema) Validate(r schema.Resource) error { return gvs.actual.Unify(rv.Value()).Validate(cue.Concrete(true)) } -// ApplyDefaults returns a new, concrete copy of the Resource with all paths -// that are 1) missing in the Resource AND 2) specified by the schema, -// filled with default values specified by the schema. -func (gvs *genericVersionedSchema) ApplyDefaults(r schema.Resource) (schema.Resource, error) { - rv, err := rt.Compile("resource", r.Value) - if err != nil { - return r, err - } - rvUnified := rv.Value().Unify(gvs.CUE()) - re, err := convertCUEValueToString(rvUnified) - if err != nil { - return r, err - } - return schema.Resource{Value: re}, nil -} - -func convertCUEValueToString(inputCUE cue.Value) (string, error) { - re, err := cuejson.Marshal(inputCUE) - if err != nil { - return re, err - } - - result := []byte(re) - result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1) - result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1) - result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1) - return string(result), nil -} - -// TrimDefaults returns a new, concrete copy of the Resource where all paths -// in the where the values at those paths are the same as the default value -// given in the schema. -func (gvs *genericVersionedSchema) TrimDefaults(r schema.Resource) (schema.Resource, error) { - rvInstance, err := rt.Compile("resource", r.Value) - if err != nil { - return r, err - } - rv, _, err := removeDefaultHelper(gvs.CUE(), rvInstance.Value()) - if err != nil { - return r, err - } - re, err := convertCUEValueToString(rv) - fmt.Println("the trimed fields would be: ", re) - if err != nil { - return r, err - } - return schema.Resource{Value: re}, nil -} - -func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) { - // Since for now, panel definition is open validation, - // we need to loop on the input CUE for trimming - rvInstance, err := rt.Compile("resource", []byte{}) - if err != nil { - return input, false, err - } - rv := rvInstance.Value() - - switch inputdef.IncompleteKind() { - case cue.StructKind: - // Get all fields including optional fields - iter, err := inputdef.Fields(cue.Optional(true)) - if err != nil { - return rv, false, err - } - keySet := make(map[string]bool) - for iter.Next() { - lable, _ := iter.Value().Label() - keySet[lable] = true - lv := input.LookupPath(cue.MakePath(cue.Str(lable))) - if err != nil { - continue - } - if lv.Exists() { - re, isEqual, err := removeDefaultHelper(iter.Value(), lv) - if err == nil && !isEqual { - rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re) - } - } - } - // Get all the fields that are not defined in schema yet for panel - iter, err = input.Fields() - if err != nil { - return rv, false, err - } - for iter.Next() { - lable, _ := iter.Value().Label() - if exists := keySet[lable]; !exists { - rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value()) - } - } - return rv, false, nil - case cue.ListKind: - val, _ := inputdef.Default() - err1 := input.Subsume(val) - err2 := val.Subsume(input) - if val.IsConcrete() && err1 == nil && err2 == nil { - return rv, true, nil - } - ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex)) - if ele.IncompleteKind() == cue.BottomKind { - return rv, true, nil - } - - iter, err := input.List() - if err != nil { - return rv, true, nil - } - var iterlist []string - for iter.Next() { - re, isEqual, err := removeDefaultHelper(ele, iter.Value()) - if err == nil && !isEqual { - reString, err := convertCUEValueToString(re) - if err != nil { - return rv, true, nil - } - iterlist = append(iterlist, reString) - } - } - iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ",")) - liInstance, err := rt.Compile("resource", []byte(iterlistContent)) - if err != nil { - return rv, false, err - } - return liInstance.Value(), false, nil - default: - val, _ := inputdef.Default() - err1 := input.Subsume(val) - err2 := val.Subsume(input) - if val.IsConcrete() && err1 == nil && err2 == nil { - return input, true, nil - } - return input, false, nil - } -} - // CUE returns the cue.Value representing the actual schema. func (gvs *genericVersionedSchema) CUE() cue.Value { return gvs.actual diff --git a/pkg/schema/load/load_test.go b/pkg/schema/load/load_test.go index 2f68888d98c..07d6dff602f 100644 --- a/pkg/schema/load/load_test.go +++ b/pkg/schema/load/load_test.go @@ -51,7 +51,7 @@ func TestDashboardValidity(t *testing.T) { // TODO FIXME remove this once we actually have dashboard schema filled in // enough that the tests pass, lol t.Skip() - validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards", "basic")) + validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards")) dash, err := BaseDashboardFamily(p) require.NoError(t, err, "error while loading base dashboard scuemata") diff --git a/pkg/schema/load/testdata/artifacts/dashboards/basic/basic.json b/pkg/schema/load/testdata/artifacts/dashboards/basic.json similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/basic/basic.json rename to pkg/schema/load/testdata/artifacts/dashboards/basic.json diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 4035918b82c..719eb34ba94 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -1,13 +1,18 @@ package schema import ( + "bytes" "errors" "fmt" "math/bits" + "strings" "cuelang.org/go/cue" + cuejson "cuelang.org/go/pkg/encoding/json" ) +var rt = &cue.Runtime{} + // CueSchema represents a single, complete CUE-based schema that can perform // operations on Resources. // @@ -23,16 +28,6 @@ type CueSchema interface { // Validate checks that the resource is correct with respect to the schema. Validate(Resource) error - // ApplyDefaults returns a new, concrete copy of the Resource with all paths - // that are 1) missing in the Resource AND 2) specified by the schema, - // filled with default values specified by the schema. - ApplyDefaults(Resource) (Resource, error) - - // TrimDefaults returns a new, concrete copy of the Resource where all paths - // in the where the values at those paths are the same as the default value - // given in the schema. - TrimDefaults(Resource) (Resource, error) - // Migrate transforms a Resource into a new Resource that is correct with // respect to its Successor schema. It returns the transformed resource, // the schema to which the resource now conforms, and any errors that @@ -257,6 +252,142 @@ func Exact(maj, min int) SearchOption { } } +// ApplyDefaults returns a new, concrete copy of the Resource with all paths +// that are 1) missing in the Resource AND 2) specified by the schema, +// filled with default values specified by the schema. +func ApplyDefaults(r Resource, scue cue.Value) (Resource, error) { + rv, err := rt.Compile("resource", r.Value) + if err != nil { + return r, err + } + rvUnified := rv.Value().Unify(scue) + re, err := convertCUEValueToString(rvUnified) + if err != nil { + return r, err + } + return Resource{Value: re}, nil +} + +func convertCUEValueToString(inputCUE cue.Value) (string, error) { + re, err := cuejson.Marshal(inputCUE) + if err != nil { + return re, err + } + + result := []byte(re) + result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1) + result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1) + result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1) + return string(result), nil +} + +// TrimDefaults returns a new, concrete copy of the Resource where all paths +// in the where the values at those paths are the same as the default value +// given in the schema. +func TrimDefaults(r Resource, scue cue.Value) (Resource, error) { + rvInstance, err := rt.Compile("resource", r.Value) + if err != nil { + return r, err + } + rv, _, err := removeDefaultHelper(scue, rvInstance.Value()) + if err != nil { + return r, err + } + re, err := convertCUEValueToString(rv) + if err != nil { + return r, err + } + return Resource{Value: re}, nil +} + +func isCueValueEqual(inputdef cue.Value, input cue.Value) bool { + val, _ := inputdef.Default() + return input.Subsume(val) == nil && val.Subsume(input) == nil +} + +func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) { + // To include all optional fields, we need to use inputdef for iteration, + // since the lookuppath with optional field doesn't work very well + rvInstance, err := rt.Compile("resource", []byte{}) + if err != nil { + return input, false, err + } + rv := rvInstance.Value() + + switch inputdef.IncompleteKind() { + case cue.StructKind: + // Get all fields including optional fields + iter, err := inputdef.Fields(cue.Optional(true)) + if err != nil { + return rv, false, err + } + keySet := make(map[string]bool) + for iter.Next() { + lable, _ := iter.Value().Label() + keySet[lable] = true + lv := input.LookupPath(cue.MakePath(cue.Str(lable))) + if err != nil { + continue + } + if lv.Exists() { + re, isEqual, err := removeDefaultHelper(iter.Value(), lv) + if err == nil && !isEqual { + rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re) + } + } + } + // Get all the fields that are not defined in schema yet for panel + iter, err = input.Fields() + if err != nil { + return rv, false, err + } + for iter.Next() { + lable, _ := iter.Value().Label() + if exists := keySet[lable]; !exists { + rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value()) + } + } + return rv, false, nil + case cue.ListKind: + if isCueValueEqual(inputdef, input) { + return rv, true, nil + } + ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex)) + if ele.IncompleteKind() == cue.BottomKind { + return rv, true, nil + } + + iter, err := input.List() + if err != nil { + return rv, true, nil + } + + // The following code is workaround since today overwrite list element doesn't work + var iterlist []string + for iter.Next() { + re, isEqual, err := removeDefaultHelper(ele, iter.Value()) + if err == nil && !isEqual { + reString, err := convertCUEValueToString(re) + if err != nil { + return rv, true, nil + } + iterlist = append(iterlist, reString) + } + } + iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ",")) + liInstance, err := rt.Compile("resource", []byte(iterlistContent)) + if err != nil { + return rv, false, err + } + return liInstance.Value(), false, nil + default: + if isCueValueEqual(inputdef, input) { + return input, true, nil + } + return input, false, nil + } +} + // A Resource represents a concrete data object - e.g., JSON // representing a dashboard. // diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index fee087844dd..178bbb1e784 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -1,4 +1,108 @@ package schema -// TODO tests for this stuff! Everything in this package is totally generic, -// nothing is specific to Grafana +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "testing" + + "cuelang.org/go/cue" + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/txtar" +) + +var CasesDir = filepath.Join("testdata", "trimapplydefaults") + +type Case struct { + Name string + CUE string + Full string + Trimmed string +} + +func TestGenerate(t *testing.T) { + cases, err := loadCases(CasesDir) + if err != nil { + t.Fatal(err) + } + + for _, c := range cases { + t.Run(c.Name+" apply default value", func(t *testing.T) { + var r cue.Runtime + scmInstance, err := r.Compile(c.Name+".cue", c.CUE) + if err != nil { + t.Fatal(err) + } + inputResource := Resource{Value: c.Trimmed} + out, err := ApplyDefaults(inputResource, scmInstance.Value()) + if err != nil { + t.Fatal(err) + } + b := []byte(out.Value.(string)) + + if s := cmp.Diff(string(b), c.Full); s != "" { + t.Fatal(s) + } + }) + } + + for _, c := range cases { + t.Run(c.Name+" trim default value", func(t *testing.T) { + var r cue.Runtime + scmInstance, err := r.Compile(c.Name+".cue", c.CUE) + if err != nil { + t.Fatal(err) + } + inputResource := Resource{Value: c.Full} + out, err := TrimDefaults(inputResource, scmInstance.Value()) + if err != nil { + t.Fatal(err) + } + b := []byte(out.Value.(string)) + if s := cmp.Diff(string(b), c.Trimmed); s != "" { + t.Fatal(s) + } + }) + } +} + +func loadCases(dir string) ([]Case, error) { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + var cases []Case + for _, fi := range files { + file := filepath.Join(dir, fi.Name()) + a, err := txtar.ParseFile(file) + if err != nil { + return nil, err + } + + if len(a.Files) != 3 { + return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files)) + } + + fullBuffer := new(bytes.Buffer) + fullJson := a.Files[1].Data + if err := json.Compact(fullBuffer, fullJson); err != nil { + return nil, err + } + + trimBuffer := new(bytes.Buffer) + trimedJson := a.Files[2].Data + if err := json.Compact(trimBuffer, trimedJson); err != nil { + return nil, err + } + + cases = append(cases, Case{ + Name: fi.Name(), + CUE: string(a.Files[0].Data), + Full: fullBuffer.String(), + Trimmed: trimBuffer.String(), + }) + } + return cases, nil +} diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test1 b/pkg/schema/testdata/trimapplydefaults/test1 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test1 rename to pkg/schema/testdata/trimapplydefaults/test1 diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test2 b/pkg/schema/testdata/trimapplydefaults/test2 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test2 rename to pkg/schema/testdata/trimapplydefaults/test2 diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test3 b/pkg/schema/testdata/trimapplydefaults/test3 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test3 rename to pkg/schema/testdata/trimapplydefaults/test3 diff --git a/pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test4 b/pkg/schema/testdata/trimapplydefaults/test4 similarity index 100% rename from pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test4 rename to pkg/schema/testdata/trimapplydefaults/test4 diff --git a/pkg/services/ngalert/api/forked_am.go b/pkg/services/ngalert/api/forked_am.go index 5eec484ba6f..463ded6cf85 100644 --- a/pkg/services/ngalert/api/forked_am.go +++ b/pkg/services/ngalert/api/forked_am.go @@ -117,22 +117,16 @@ func (am *ForkedAMSvc) RoutePostAlertingConfig(ctx *models.ReqContext, body apim return response.Error(400, err.Error(), nil) } - backendType, err := backendType(ctx, am.DatasourceCache) + b, err := backendType(ctx, am.DatasourceCache) if err != nil { return response.Error(400, err.Error(), nil) } - payloadType := body.AlertmanagerConfig.Type() - - if backendType != payloadType { + if err := body.AlertmanagerConfig.ReceiverType().MatchesBackend(b); err != nil { return response.Error( 400, - fmt.Sprintf( - "unexpected backend type (%v) vs payload type (%v)", - backendType, - payloadType, - ), - nil, + "bad match", + err, ) } diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go index fdc9792900d..b4429602f12 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "reflect" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/models" @@ -381,6 +382,8 @@ func (c *GettableApiAlertingConfig) validate() error { hasGrafReceivers = true case AlertmanagerReceiverType: hasAMReceivers = true + default: + continue } } @@ -398,19 +401,6 @@ func (c *GettableApiAlertingConfig) validate() error { return nil } -// Type requires validate has been called and just checks the first receiver type -func (c *GettableApiAlertingConfig) Type() (backend Backend) { - for _, r := range c.Receivers { - switch r.Type() { - case GrafanaReceiverType: - return GrafanaBackend - case AlertmanagerReceiverType: - return AlertmanagerBackend - } - } - return -} - // Config is the top-level configuration for Alertmanager's config files. type Config struct { Global *config.GlobalConfig `yaml:"global,omitempty" json:"global,omitempty"` @@ -448,6 +438,8 @@ func (c *PostableApiAlertingConfig) validate() error { hasGrafReceivers = true case AlertmanagerReceiverType: hasAMReceivers = true + default: + continue } } @@ -466,22 +458,26 @@ func (c *PostableApiAlertingConfig) validate() error { } // Type requires validate has been called and just checks the first receiver type -func (c *PostableApiAlertingConfig) Type() (backend Backend) { +func (c *PostableApiAlertingConfig) ReceiverType() ReceiverType { for _, r := range c.Receivers { switch r.Type() { case GrafanaReceiverType: - return GrafanaBackend + return GrafanaReceiverType case AlertmanagerReceiverType: - return AlertmanagerBackend + return AlertmanagerReceiverType + default: + continue } } - return + return EmptyReceiverType } // AllReceivers will recursively walk a routing tree and return a list of all the // referenced receiver names. func AllReceivers(route *config.Route) (res []string) { - res = append(res, route.Receiver) + if route.Receiver != "" { + res = append(res, route.Receiver) + } for _, subRoute := range route.Routes { res = append(res, AllReceivers(subRoute)...) } @@ -494,10 +490,52 @@ type PostableGrafanaReceiver models.CreateAlertNotificationCommand type ReceiverType int const ( - GrafanaReceiverType ReceiverType = iota + GrafanaReceiverType ReceiverType = 1 << iota AlertmanagerReceiverType + EmptyReceiverType = GrafanaReceiverType | AlertmanagerReceiverType ) +func (r ReceiverType) String() string { + switch r { + case GrafanaReceiverType: + return "grafana" + case AlertmanagerReceiverType: + return "alertmanager" + case EmptyReceiverType: + return "empty" + default: + return "unknown" + } +} + +// Can determines whether a receiver type can implement another receiver type. +// This is useful as receivers with just names but no contact points +// are valid in all backends. +func (r ReceiverType) Can(other ReceiverType) bool { return r&other != 0 } + +// MatchesBackend determines if a config payload can be sent to a particular backend type +func (r ReceiverType) MatchesBackend(backend Backend) error { + msg := func(backend Backend, receiver ReceiverType) error { + return fmt.Errorf( + "unexpected backend type (%s) for receiver type (%s)", + backend.String(), + receiver.String(), + ) + } + var ok bool + switch backend { + case GrafanaBackend: + ok = r.Can(GrafanaReceiverType) + case AlertmanagerBackend: + ok = r.Can(AlertmanagerReceiverType) + default: + } + if !ok { + return msg(backend, r) + } + return nil +} + type GettableApiReceiver struct { config.Receiver `yaml:",inline"` GettableGrafanaReceivers `yaml:",inline"` @@ -554,25 +592,14 @@ type PostableApiReceiver struct { } func (r *PostableApiReceiver) UnmarshalYAML(unmarshal func(interface{}) error) error { - var grafanaReceivers PostableGrafanaReceivers - if err := unmarshal(&grafanaReceivers); err != nil { + if err := unmarshal(&r.PostableGrafanaReceivers); err != nil { return err } - r.PostableGrafanaReceivers = grafanaReceivers - var cfg config.Receiver - if err := unmarshal(&cfg); err != nil { + if err := unmarshal(&r.Receiver); err != nil { return err } - r.Name = cfg.Name - r.EmailConfigs = cfg.EmailConfigs - r.PagerdutyConfigs = cfg.PagerdutyConfigs - r.SlackConfigs = cfg.SlackConfigs - r.WebhookConfigs = cfg.WebhookConfigs - r.OpsGenieConfigs = cfg.OpsGenieConfigs - r.WechatConfigs = cfg.WechatConfigs - r.PushoverConfigs = cfg.PushoverConfigs - r.VictorOpsConfigs = cfg.VictorOpsConfigs + return nil } @@ -617,6 +644,13 @@ func (r *PostableApiReceiver) Type() ReceiverType { if len(r.PostableGrafanaReceivers.GrafanaManagedReceivers) > 0 { return GrafanaReceiverType } + + cpy := r.Receiver + cpy.Name = "" + if reflect.ValueOf(cpy).IsZero() { + return EmptyReceiverType + } + return AlertmanagerReceiverType } diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go index 0e0bca32008..529e69c56b2 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go @@ -69,6 +69,50 @@ func Test_ApiReceiver_Marshaling(t *testing.T) { } } +func Test_APIReceiverType(t *testing.T) { + for _, tc := range []struct { + desc string + input PostableApiReceiver + expected ReceiverType + }{ + { + desc: "empty", + input: PostableApiReceiver{ + Receiver: config.Receiver{ + Name: "foo", + }, + }, + expected: EmptyReceiverType, + }, + { + desc: "am", + input: PostableApiReceiver{ + Receiver: config.Receiver{ + Name: "foo", + EmailConfigs: []*config.EmailConfig{{}}, + }, + }, + expected: AlertmanagerReceiverType, + }, + { + desc: "graf", + input: PostableApiReceiver{ + Receiver: config.Receiver{ + Name: "foo", + }, + PostableGrafanaReceivers: PostableGrafanaReceivers{ + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{}}, + }, + }, + expected: GrafanaReceiverType, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.expected, tc.input.Type()) + }) + } +} + func Test_AllReceivers(t *testing.T) { input := &config.Route{ Receiver: "foo", @@ -88,6 +132,10 @@ func Test_AllReceivers(t *testing.T) { } require.Equal(t, []string{"foo", "bar", "bazz", "buzz"}, AllReceivers(input)) + + // test empty + var empty []string + require.Equal(t, empty, AllReceivers(&config.Route{})) } func Test_ApiAlertingConfig_Marshaling(t *testing.T) { @@ -405,3 +453,113 @@ func Test_GettableUserConfigRoundtrip(t *testing.T) { require.Nil(t, err) require.Equal(t, string(yamlEncoded), string(out)) } + +func Test_ReceiverCompatibility(t *testing.T) { + for _, tc := range []struct { + desc string + a, b ReceiverType + expected bool + }{ + { + desc: "grafana=grafana", + a: GrafanaReceiverType, + b: GrafanaReceiverType, + expected: true, + }, + { + desc: "am=am", + a: AlertmanagerReceiverType, + b: AlertmanagerReceiverType, + expected: true, + }, + { + desc: "empty=grafana", + a: EmptyReceiverType, + b: AlertmanagerReceiverType, + expected: true, + }, + { + desc: "empty=am", + a: EmptyReceiverType, + b: AlertmanagerReceiverType, + expected: true, + }, + { + desc: "empty=empty", + a: EmptyReceiverType, + b: EmptyReceiverType, + expected: true, + }, + { + desc: "graf!=am", + a: GrafanaReceiverType, + b: AlertmanagerReceiverType, + expected: false, + }, + { + desc: "am!=graf", + a: AlertmanagerReceiverType, + b: GrafanaReceiverType, + expected: false, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + require.Equal(t, tc.expected, tc.a.Can(tc.b)) + }) + } +} + +func Test_ReceiverMatchesBackend(t *testing.T) { + for _, tc := range []struct { + desc string + rec ReceiverType + b Backend + err bool + }{ + { + desc: "graf=graf", + rec: GrafanaReceiverType, + b: GrafanaBackend, + err: false, + }, + { + desc: "empty=graf", + rec: EmptyReceiverType, + b: GrafanaBackend, + err: false, + }, + { + desc: "am=am", + rec: AlertmanagerReceiverType, + b: AlertmanagerBackend, + err: false, + }, + { + desc: "empty=am", + rec: EmptyReceiverType, + b: AlertmanagerBackend, + err: false, + }, + { + desc: "graf!=am", + rec: GrafanaReceiverType, + b: AlertmanagerBackend, + err: true, + }, + { + desc: "am!=ruler", + rec: GrafanaReceiverType, + b: LoTexRulerBackend, + err: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + err := tc.rec.MatchesBackend(tc.b) + if tc.err { + require.NotNil(t, err) + } else { + require.Nil(t, err) + } + }) + } +} diff --git a/pkg/services/ngalert/models/instance.go b/pkg/services/ngalert/models/instance.go index dd14f8a2bb4..f3e6ae19941 100644 --- a/pkg/services/ngalert/models/instance.go +++ b/pkg/services/ngalert/models/instance.go @@ -7,8 +7,8 @@ import ( // AlertInstance represents a single alert instance. type AlertInstance struct { - RuleOrgID int64 `xorm:"def_org_id"` - RuleUID string `xorm:"def_uid"` + RuleOrgID int64 `xorm:"rule_org_id"` + RuleUID string `xorm:"rule_uid"` Labels InstanceLabels LabelsHash string CurrentState InstanceStateType @@ -73,14 +73,10 @@ type ListAlertInstancesQuery struct { Result []*ListAlertInstancesQueryResult } -type FetchUniqueOrgIdsQuery struct { - Result []*FetchUniqueOrgIdsQueryResult -} - // ListAlertInstancesQueryResult represents the result of listAlertInstancesQuery. type ListAlertInstancesQueryResult struct { - RuleOrgID int64 `xorm:"def_org_id" json:"definitionOrgId"` - RuleDefinitionUID string `xorm:"def_uid" json:"definitionUid"` + RuleOrgID int64 `xorm:"rule_org_id" json:"ruleOrgId"` + RuleUID string `xorm:"rule_uid" json:"ruleUid"` Labels InstanceLabels `json:"labels"` LabelsHash string `json:"labeHash"` CurrentState InstanceStateType `json:"currentState"` @@ -89,11 +85,7 @@ type ListAlertInstancesQueryResult struct { LastEvalTime time.Time `json:"lastEvalTime"` } -type FetchUniqueOrgIdsQueryResult struct { - DefinitionOrgID int64 `xorm:"def_org_id" json:"definitionOrgId"` -} - -// ValidateAlertInstance validates that the alert instance contains an alert definition id, +// ValidateAlertInstance validates that the alert instance contains an alert rule id, // and state. func ValidateAlertInstance(alertInstance *AlertInstance) error { if alertInstance == nil { @@ -101,11 +93,11 @@ func ValidateAlertInstance(alertInstance *AlertInstance) error { } if alertInstance.RuleOrgID == 0 { - return fmt.Errorf("alert instance is invalid due to missing alert definition organisation") + return fmt.Errorf("alert instance is invalid due to missing alert rule organisation") } if alertInstance.RuleUID == "" { - return fmt.Errorf("alert instance is invalid due to missing alert definition uid") + return fmt.Errorf("alert instance is invalid due to missing alert rule uid") } if !alertInstance.CurrentState.IsValid() { diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 85d5bb22c90..dc22575e252 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -260,8 +260,6 @@ func (am *Alertmanager) SyncAndApplyConfigFromDatabase() error { return nil } -const defaultTemplate = "templates/default.tmpl" - // applyConfig applies a new configuration by re-initializing all components using the configuration provided. // It is not safe to call concurrently. func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig []byte) error { @@ -279,6 +277,12 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig if md5.Sum(am.config) != md5.Sum(rawConfig) { configChanged = true } + + if cfg.TemplateFiles == nil { + cfg.TemplateFiles = map[string]string{} + } + cfg.TemplateFiles["__default__.tmpl"] = channels.DefaultTemplateString + // next, we need to make sure we persist the templates to disk. paths, templatesChanged, err := PersistTemplates(cfg, am.WorkingDirPath()) if err != nil { @@ -291,8 +295,6 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig return nil } - paths = append([]string{defaultTemplate}, paths...) - // With the templates persisted, create the template list using the paths. tmpl, err := template.FromGlobs(paths...) if err != nil { diff --git a/pkg/services/ngalert/notifier/channels/default_template.go b/pkg/services/ngalert/notifier/channels/default_template.go new file mode 100644 index 00000000000..da9f7570414 --- /dev/null +++ b/pkg/services/ngalert/notifier/channels/default_template.go @@ -0,0 +1,51 @@ +package channels + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/prometheus/alertmanager/template" + "github.com/stretchr/testify/require" +) + +const DefaultTemplateString = ` +{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }} + +{{ define "__text_alert_list" }}{{ range . }}Labels: +{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} +{{ end }}Annotations: +{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} +{{ end }}Source: {{ .GeneratorURL }} +{{ end }}{{ end }} + +{{ define "default.title" }}{{ template "__subject" . }}{{ end }} + +{{ define "default.message" }}{{ if gt (len .Alerts.Firing) 0 }} +**Firing** +{{ template "__text_alert_list" .Alerts.Firing }} + +{{ end }} +{{ if gt (len .Alerts.Resolved) 0 }} +**Resolved** +{{ template "__text_alert_list" .Alerts.Resolved }} +{{ end }} +{{ end }} +` + +func templateForTests(t *testing.T) *template.Template { + f, err := ioutil.TempFile("/tmp", "template") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(f.Name())) + }) + + _, err = f.WriteString(DefaultTemplateString) + require.NoError(t, err) + + tmpl, err := template.FromGlobs(f.Name()) + require.NoError(t, err) + + return tmpl +} diff --git a/pkg/services/ngalert/notifier/channels/dingding_test.go b/pkg/services/ngalert/notifier/channels/dingding_test.go index f45056edc06..2c4c169261f 100644 --- a/pkg/services/ngalert/notifier/channels/dingding_test.go +++ b/pkg/services/ngalert/notifier/channels/dingding_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/prometheus/alertmanager/notify" - "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -20,8 +19,7 @@ import ( ) func TestDingdingNotifier(t *testing.T) { - tmpl, err := template.FromGlobs("templates/default.tmpl") - require.NoError(t, err) + tmpl := templateForTests(t) externalURL, err := url.Parse("http://localhost") require.NoError(t, err) diff --git a/pkg/services/ngalert/notifier/channels/pagerduty.go b/pkg/services/ngalert/notifier/channels/pagerduty.go index 0c008a17dc7..9b1539582a6 100644 --- a/pkg/services/ngalert/notifier/channels/pagerduty.go +++ b/pkg/services/ngalert/notifier/channels/pagerduty.go @@ -58,8 +58,8 @@ func NewPagerdutyNotifier(model *models.AlertNotification, t *template.Template) NotifierBase: old_notifiers.NewNotifierBase(model), Key: key, CustomDetails: map[string]string{ - "firing": `{{ template "pagerduty.default.instances" .Alerts.Firing }}`, - "resolved": `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`, + "firing": `{{ template "__text_alert_list" .Alerts.Firing }}`, + "resolved": `{{ template "__text_alert_list" .Alerts.Resolved }}`, "num_firing": `{{ .Alerts.Firing | len }}`, "num_resolved": `{{ .Alerts.Resolved | len }}`, }, @@ -67,7 +67,7 @@ func NewPagerdutyNotifier(model *models.AlertNotification, t *template.Template) Class: model.Settings.Get("class").MustString("default"), Component: model.Settings.Get("component").MustString("Grafana"), Group: model.Settings.Get("group").MustString("default"), - Summary: model.Settings.Get("summary").MustString(`{{ template "pagerduty.default.description" .}}`), + Summary: model.Settings.Get("summary").MustString(`{{ template "default.title" . }}`), tmpl: t, log: log.New("alerting.notifier." + model.Name), }, nil diff --git a/pkg/services/ngalert/notifier/channels/pagerduty_test.go b/pkg/services/ngalert/notifier/channels/pagerduty_test.go index 466f3dae960..49f821e8f5c 100644 --- a/pkg/services/ngalert/notifier/channels/pagerduty_test.go +++ b/pkg/services/ngalert/notifier/channels/pagerduty_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/prometheus/alertmanager/notify" - "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -21,8 +20,7 @@ import ( ) func TestPagerdutyNotifier(t *testing.T) { - tmpl, err := template.FromGlobs("templates/default.tmpl") - require.NoError(t, err) + tmpl := templateForTests(t) externalURL, err := url.Parse("http://localhost") require.NoError(t, err) diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index de02f75455f..b1ffc75fced 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -121,8 +121,8 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S IconEmoji: model.Settings.Get("icon_emoji").MustString(), IconURL: model.Settings.Get("icon_url").MustString(), Token: token, - Text: model.Settings.Get("text").MustString(`{{ template "slack.default.text" . }}`), - Title: model.Settings.Get("title").MustString(`{{ template "slack.default.title" . }}`), + Text: model.Settings.Get("text").MustString(`{{ template "default.message" . }}`), + Title: model.Settings.Get("title").MustString(`{{ template "default.title" . }}`), log: log.New("alerting.notifier.slack"), tmpl: t, }, nil diff --git a/pkg/services/ngalert/notifier/channels/slack_test.go b/pkg/services/ngalert/notifier/channels/slack_test.go index 02591d2aca6..497516f174b 100644 --- a/pkg/services/ngalert/notifier/channels/slack_test.go +++ b/pkg/services/ngalert/notifier/channels/slack_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/prometheus/alertmanager/notify" - "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -22,8 +21,7 @@ import ( ) func TestSlackNotifier(t *testing.T) { - tmpl, err := template.FromGlobs("templates/default.tmpl") - require.NoError(t, err) + tmpl := templateForTests(t) externalURL, err := url.Parse("http://localhost") require.NoError(t, err) @@ -60,7 +58,7 @@ func TestSlackNotifier(t *testing.T) { { Title: "[FIRING:1] (val1)", TitleLink: "http:/localhost/alerting/list", - Text: "", + Text: "\n**Firing**\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSource: \n\n\n\n\n", Fallback: "[FIRING:1] (val1)", Fields: nil, Footer: "Grafana v", @@ -96,7 +94,7 @@ func TestSlackNotifier(t *testing.T) { { Title: "[FIRING:1] (val1)", TitleLink: "http:/localhost/alerting/list", - Text: "", + Text: "\n**Firing**\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSource: \n\n\n\n\n", Fallback: "[FIRING:1] (val1)", Fields: nil, Footer: "Grafana v", @@ -139,7 +137,7 @@ func TestSlackNotifier(t *testing.T) { { Title: "2 firing, 0 resolved", TitleLink: "http:/localhost/alerting/list", - Text: "", + Text: "\n**Firing**\nLabels:\n - alertname = alert1\n - lbl1 = val1\nAnnotations:\n - ann1 = annv1\nSource: \nLabels:\n - alertname = alert1\n - lbl1 = val2\nAnnotations:\n - ann1 = annv2\nSource: \n\n\n\n\n", Fallback: "2 firing, 0 resolved", Fields: nil, Footer: "Grafana v", diff --git a/pkg/services/ngalert/notifier/channels/teams_test.go b/pkg/services/ngalert/notifier/channels/teams_test.go index f032e858470..83fa93d18ad 100644 --- a/pkg/services/ngalert/notifier/channels/teams_test.go +++ b/pkg/services/ngalert/notifier/channels/teams_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/prometheus/alertmanager/notify" - "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -20,8 +19,7 @@ import ( ) func TestTeamsNotifier(t *testing.T) { - tmpl, err := template.FromGlobs("templates/default.tmpl") - require.NoError(t, err) + tmpl := templateForTests(t) externalURL, err := url.Parse("http://localhost") require.NoError(t, err) diff --git a/pkg/services/ngalert/notifier/channels/telegram_test.go b/pkg/services/ngalert/notifier/channels/telegram_test.go index 0419224419a..c11a09a5ca2 100644 --- a/pkg/services/ngalert/notifier/channels/telegram_test.go +++ b/pkg/services/ngalert/notifier/channels/telegram_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/prometheus/alertmanager/notify" - "github.com/prometheus/alertmanager/template" "github.com/prometheus/alertmanager/types" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" @@ -18,8 +17,7 @@ import ( ) func TestTelegramNotifier(t *testing.T) { - tmpl, err := template.FromGlobs("templates/default.tmpl") - require.NoError(t, err) + tmpl := templateForTests(t) externalURL, err := url.Parse("http://localhost") require.NoError(t, err) diff --git a/pkg/services/ngalert/notifier/channels/templates/default.tmpl b/pkg/services/ngalert/notifier/channels/templates/default.tmpl deleted file mode 100644 index 73c7fc08416..00000000000 --- a/pkg/services/ngalert/notifier/channels/templates/default.tmpl +++ /dev/null @@ -1,230 +0,0 @@ -{{ define "__alertmanager" }}Alertmanager{{ end }} -{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver | urlquery }}{{ end }} - -{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }} -{{ define "__description" }}{{ end }} - -{{ define "__text_alert_list" }}{{ range . }}Labels: -{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} -{{ end }}Annotations: -{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} -{{ end }}Source: {{ .GeneratorURL }} -{{ end }}{{ end }} - -{{ define "default.title" }}{{ template "__subject" . }}{{ end }} -{{ define "default.message" }}{{ if gt (len .Alerts.Firing) 0 }} -**Firing** -{{ template "__text_alert_list" .Alerts.Firing }} - -{{ end }} -{{ if gt (len .Alerts.Resolved) 0 }} -**Resolved** -{{ template "__text_alert_list" .Alerts.Resolved }} -{{ end }} -{{ end }} - -{{ define "slack.default.title" }}{{ template "default.title" . }}{{ end }} -{{ define "slack.default.username" }}{{ template "__alertmanager" . }}{{ end }} -{{ define "slack.default.fallback" }}{{ template "slack.default.title" . }} | {{ template "slack.default.titlelink" . }}{{ end }} -{{ define "slack.default.callbackid" }}{{ end }} -{{ define "slack.default.pretext" }}{{ end }} -{{ define "slack.default.titlelink" }}{{ template "__alertmanagerURL" . }}{{ end }} -{{ define "slack.default.iconemoji" }}{{ end }} -{{ define "slack.default.iconurl" }}{{ end }} -{{ define "slack.default.text" }}{{ end }} -{{ define "slack.default.footer" }}{{ end }} - - -{{ define "pagerduty.default.description" }}{{ template "default.title" . }}{{ end }} -{{ define "pagerduty.default.client" }}{{ template "__alertmanager" . }}{{ end }} -{{ define "pagerduty.default.clientURL" }}{{ template "__alertmanagerURL" . }}{{ end }} -{{ define "pagerduty.default.instances" }}{{ template "__text_alert_list" . }}{{ end }} - - -{{ define "opsgenie.default.message" }}{{ template "__subject" . }}{{ end }} -{{ define "opsgenie.default.description" }}{{ .CommonAnnotations.SortedPairs.Values | join " " }} -{{ if gt (len .Alerts.Firing) 0 -}} -Alerts Firing: -{{ template "__text_alert_list" .Alerts.Firing }} -{{- end }} -{{ if gt (len .Alerts.Resolved) 0 -}} -Alerts Resolved: -{{ template "__text_alert_list" .Alerts.Resolved }} -{{- end }} -{{- end }} -{{ define "opsgenie.default.source" }}{{ template "__alertmanagerURL" . }}{{ end }} - - -{{ define "wechat.default.message" }}{{ template "__subject" . }} -{{ .CommonAnnotations.SortedPairs.Values | join " " }} -{{ if gt (len .Alerts.Firing) 0 -}} -Alerts Firing: -{{ template "__text_alert_list" .Alerts.Firing }} -{{- end }} -{{ if gt (len .Alerts.Resolved) 0 -}} -Alerts Resolved: -{{ template "__text_alert_list" .Alerts.Resolved }} -{{- end }} -AlertmanagerUrl: -{{ template "__alertmanagerURL" . }} -{{- end }} -{{ define "wechat.default.to_user" }}{{ end }} -{{ define "wechat.default.to_party" }}{{ end }} -{{ define "wechat.default.to_tag" }}{{ end }} -{{ define "wechat.default.agent_id" }}{{ end }} - - - -{{ define "victorops.default.state_message" }}{{ .CommonAnnotations.SortedPairs.Values | join " " }} -{{ if gt (len .Alerts.Firing) 0 -}} -Alerts Firing: -{{ template "__text_alert_list" .Alerts.Firing }} -{{- end }} -{{ if gt (len .Alerts.Resolved) 0 -}} -Alerts Resolved: -{{ template "__text_alert_list" .Alerts.Resolved }} -{{- end }} -{{- end }} -{{ define "victorops.default.entity_display_name" }}{{ template "__subject" . }}{{ end }} -{{ define "victorops.default.monitoring_tool" }}{{ template "__alertmanager" . }}{{ end }} - -{{ define "email.default.subject" }}{{ template "__subject" . }}{{ end }} -{{ define "email.default.html" }} - - - - - - -{{ template "__subject" . }} - - - - - - - - - - - -
-
- - - - - - - -
- {{ .Alerts | len }} alert{{ if gt (len .Alerts) 1 }}s{{ end }} for {{ range .GroupLabels.SortedPairs }} - {{ .Name }}={{ .Value }} - {{ end }} -
- - - - - {{ if gt (len .Alerts.Firing) 0 }} - - - - {{ end }} - {{ range .Alerts.Firing }} - - - - {{ end }} - - {{ if gt (len .Alerts.Resolved) 0 }} - {{ if gt (len .Alerts.Firing) 0 }} - - - - {{ end }} - - - - {{ end }} - {{ range .Alerts.Resolved }} - - - - {{ end }} -
- View in {{ template "__alertmanager" . }} -
- [{{ .Alerts.Firing | len }}] Firing -
- Labels
- {{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}
{{ end }} - {{ if gt (len .Annotations) 0 }}Annotations
{{ end }} - {{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}
{{ end }} - Source
-
-
-
-
-
- [{{ .Alerts.Resolved | len }}] Resolved -
- Labels
- {{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}
{{ end }} - {{ if gt (len .Annotations) 0 }}Annotations
{{ end }} - {{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}
{{ end }} - Source
-
-
- -
-
- - - - -{{ end }} - -{{ define "pushover.default.title" }}{{ template "__subject" . }}{{ end }} -{{ define "pushover.default.message" }}{{ .CommonAnnotations.SortedPairs.Values | join " " }} -{{ if gt (len .Alerts.Firing) 0 }} -Alerts Firing: -{{ template "__text_alert_list" .Alerts.Firing }} -{{ end }} -{{ if gt (len .Alerts.Resolved) 0 }} -Alerts Resolved: -{{ template "__text_alert_list" .Alerts.Resolved }} -{{ end }} -{{ end }} -{{ define "pushover.default.url" }}{{ template "__alertmanagerURL" . }}{{ end }} diff --git a/pkg/services/ngalert/notifier/channels/webhook_test.go b/pkg/services/ngalert/notifier/channels/webhook_test.go index ee8648c0fd8..861ace71600 100644 --- a/pkg/services/ngalert/notifier/channels/webhook_test.go +++ b/pkg/services/ngalert/notifier/channels/webhook_test.go @@ -19,8 +19,7 @@ import ( ) func TestWebhookNotifier(t *testing.T) { - tmpl, err := template.FromGlobs("templates/default.tmpl") - require.NoError(t, err) + tmpl := templateForTests(t) externalURL, err := url.Parse("http://localhost") require.NoError(t, err) diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index bc87c48e7c4..b7f2490176f 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -305,16 +305,19 @@ func (sch *schedule) Ticker(grafanaCtx context.Context, stateManager *state.Mana sch.registry.del(key) } case <-grafanaCtx.Done(): - err := dispatcherGroup.Wait() - orgIdsCmd := models.FetchUniqueOrgIdsQuery{} - if err := sch.instanceStore.FetchOrgIds(&orgIdsCmd); err != nil { + waitErr := dispatcherGroup.Wait() + + orgIds, err := sch.instanceStore.FetchOrgIds() + if err != nil { sch.log.Error("unable to fetch orgIds", "msg", err.Error()) } - for _, v := range orgIdsCmd.Result { - sch.saveAlertStates(stateManager.GetAll(v.DefinitionOrgID)) + + for _, v := range orgIds { + sch.saveAlertStates(stateManager.GetAll(v)) } + stateManager.Close() - return err + return waitErr } } } @@ -346,16 +349,16 @@ func (sch *schedule) WarmStateCache(st *state.Manager) { sch.log.Info("warming cache for startup") st.ResetCache() - orgIdsCmd := models.FetchUniqueOrgIdsQuery{} - if err := sch.instanceStore.FetchOrgIds(&orgIdsCmd); err != nil { + orgIds, err := sch.instanceStore.FetchOrgIds() + if err != nil { sch.log.Error("unable to fetch orgIds", "msg", err.Error()) } var states []*state.State - for _, orgIdResult := range orgIdsCmd.Result { + for _, orgId := range orgIds { // Get Rules ruleCmd := models.ListAlertRulesQuery{ - OrgID: orgIdResult.DefinitionOrgID, + OrgID: orgId, } if err := sch.ruleStore.GetOrgAlertRules(&ruleCmd); err != nil { sch.log.Error("unable to fetch previous state", "msg", err.Error()) @@ -368,16 +371,16 @@ func (sch *schedule) WarmStateCache(st *state.Manager) { // Get Instances cmd := models.ListAlertInstancesQuery{ - RuleOrgID: orgIdResult.DefinitionOrgID, + RuleOrgID: orgId, } if err := sch.instanceStore.ListAlertInstances(&cmd); err != nil { sch.log.Error("unable to fetch previous state", "msg", err.Error()) } for _, entry := range cmd.Result { - ruleForEntry, ok := ruleByUID[entry.RuleDefinitionUID] + ruleForEntry, ok := ruleByUID[entry.RuleUID] if !ok { - sch.log.Error("rule not found for instance, ignoring", "rule", entry.RuleDefinitionUID) + sch.log.Error("rule not found for instance, ignoring", "rule", entry.RuleUID) continue } @@ -387,7 +390,7 @@ func (sch *schedule) WarmStateCache(st *state.Manager) { sch.log.Error("error getting cacheId for entry", "msg", err.Error()) } stateForEntry := &state.State{ - AlertRuleUID: entry.RuleDefinitionUID, + AlertRuleUID: entry.RuleUID, OrgID: entry.RuleOrgID, CacheId: cacheId, Labels: lbs, diff --git a/pkg/services/ngalert/store/alert_rule.go b/pkg/services/ngalert/store/alert_rule.go index 5b1e7cf2707..9a7e22c3561 100644 --- a/pkg/services/ngalert/store/alert_rule.go +++ b/pkg/services/ngalert/store/alert_rule.go @@ -80,7 +80,7 @@ func (st DBstore) DeleteAlertRuleByUID(orgID int64, ruleUID string) error { return err } - _, err = sess.Exec("DELETE FROM alert_instance WHERE def_org_id = ? AND def_uid = ?", orgID, ruleUID) + _, err = sess.Exec("DELETE FROM alert_instance WHERE rule_org_id = ? AND rule_uid = ?", orgID, ruleUID) if err != nil { return err } @@ -109,7 +109,7 @@ func (st DBstore) DeleteNamespaceAlertRules(orgID int64, namespaceUID string) ([ return err } - if _, err := sess.Exec(`DELETE FROM alert_instance WHERE def_org_id = ? AND def_uid NOT IN ( + if _, err := sess.Exec(`DELETE FROM alert_instance WHERE rule_org_id = ? AND rule_uid NOT IN ( SELECT uid FROM alert_rule where org_id = ? )`, orgID, orgID); err != nil { return err @@ -146,7 +146,7 @@ func (st DBstore) DeleteRuleGroupAlertRules(orgID int64, namespaceUID string, ru return err } - if _, err := sess.Exec(`DELETE FROM alert_instance WHERE def_org_id = ? AND def_uid NOT IN ( + if _, err := sess.Exec(`DELETE FROM alert_instance WHERE rule_org_id = ? AND rule_uid NOT IN ( SELECT uid FROM alert_rule where org_id = ? )`, orgID, orgID); err != nil { return err @@ -161,7 +161,7 @@ func (st DBstore) DeleteRuleGroupAlertRules(orgID int64, namespaceUID string, ru // DeleteAlertInstanceByRuleUID is a handler for deleting alert instances by alert rule UID when a rule has been updated func (st DBstore) DeleteAlertInstancesByRuleUID(orgID int64, ruleUID string) error { return st.SQLStore.WithTransactionalDbSession(context.Background(), func(sess *sqlstore.DBSession) error { - _, err := sess.Exec("DELETE FROM alert_instance WHERE def_org_id = ? AND def_uid = ?", orgID, ruleUID) + _, err := sess.Exec("DELETE FROM alert_instance WHERE rule_org_id = ? AND rule_uid = ?", orgID, ruleUID) if err != nil { return err } diff --git a/pkg/services/ngalert/store/database.go b/pkg/services/ngalert/store/database.go index f5fcf59a4fc..c11dd09455f 100644 --- a/pkg/services/ngalert/store/database.go +++ b/pkg/services/ngalert/store/database.go @@ -14,14 +14,6 @@ var TimeNow = time.Now // AlertDefinitionMaxTitleLength is the maximum length of the alert definition title const AlertDefinitionMaxTitleLength = 190 -// Store is the interface for persisting alert definitions and instances -type Store interface { - GetAlertInstance(*models.GetAlertInstanceQuery) error - ListAlertInstances(*models.ListAlertInstancesQuery) error - SaveAlertInstance(*models.SaveAlertInstanceCommand) error - FetchOrgIds(cmd *models.FetchUniqueOrgIdsQuery) error -} - // AlertingStore is the database interface used by the Alertmanager service. type AlertingStore interface { GetLatestAlertmanagerConfiguration(*models.GetLatestAlertmanagerConfigurationQuery) error diff --git a/pkg/services/ngalert/store/database_mig.go b/pkg/services/ngalert/store/database_mig.go index a8bf9503899..9a77850996c 100644 --- a/pkg/services/ngalert/store/database_mig.go +++ b/pkg/services/ngalert/store/database_mig.go @@ -52,6 +52,8 @@ func AddAlertDefinitionMigrations(mg *migrator.Migrator, defaultIntervalSeconds mg.AddMigration("Add column paused in alert_definition", migrator.NewAddColumnMigration(alertDefinition, &migrator.Column{ Name: "paused", Type: migrator.DB_Bool, Nullable: false, Default: "0", })) + + mg.AddMigration("drop alert_definition table", migrator.NewDropTableMigration("alert_definition")) } // AddAlertDefinitionMigrations should not be modified. @@ -84,6 +86,7 @@ func AddAlertDefinitionVersionMigrations(mg *migrator.Migrator) { mg.AddMigration("alter alert_definition_version table data column to mediumtext in mysql", migrator.NewRawSQLMigration(""). Mysql("ALTER TABLE alert_definition_version MODIFY data MEDIUMTEXT;")) + mg.AddMigration("drop alert_definition_version table", migrator.NewDropTableMigration("alert_definition_version")) } func AlertInstanceMigration(mg *migrator.Migrator) { @@ -112,6 +115,24 @@ func AlertInstanceMigration(mg *migrator.Migrator) { mg.AddMigration("add column current_state_end to alert_instance", migrator.NewAddColumnMigration(alertInstance, &migrator.Column{ Name: "current_state_end", Type: migrator.DB_BigInt, Nullable: false, Default: "0", })) + + mg.AddMigration("remove index def_org_id, def_uid, current_state on alert_instance", migrator.NewDropIndexMigration(alertInstance, alertInstance.Indices[0])) + mg.AddMigration("remove index def_org_id, current_state on alert_instance", migrator.NewDropIndexMigration(alertInstance, alertInstance.Indices[1])) + + mg.AddMigration("rename def_org_id to rule_org_id in alert_instance", migrator.NewRawSQLMigration(""). + Default("ALTER TABLE alert_instance RENAME COLUMN def_org_id TO rule_org_id;"). + Mysql("ALTER TABLE alert_instance CHANGE def_org_id rule_org_id BIGINT;")) + + mg.AddMigration("rename def_uid to rule_uid in alert_instance", migrator.NewRawSQLMigration(""). + Default("ALTER TABLE alert_instance RENAME COLUMN def_uid TO rule_uid;"). + Mysql("ALTER TABLE alert_instance CHANGE def_uid rule_uid VARCHAR(40);")) + + mg.AddMigration("add index rule_org_id, rule_uid, current_state on alert_instance", migrator.NewAddIndexMigration(alertInstance, &migrator.Index{ + Cols: []string{"rule_org_id", "rule_uid", "current_state"}, Type: migrator.IndexType, + })) + mg.AddMigration("add index rule_org_id, current_state on alert_instance", migrator.NewAddIndexMigration(alertInstance, &migrator.Index{ + Cols: []string{"rule_org_id", "current_state"}, Type: migrator.IndexType, + })) } func AddAlertRuleMigrations(mg *migrator.Migrator, defaultIntervalSeconds int64) { diff --git a/pkg/services/ngalert/store/instance_database.go b/pkg/services/ngalert/store/instance_database.go index 9b796fd5bc1..5c4bc0d7b1e 100644 --- a/pkg/services/ngalert/store/instance_database.go +++ b/pkg/services/ngalert/store/instance_database.go @@ -13,7 +13,7 @@ type InstanceStore interface { GetAlertInstance(cmd *models.GetAlertInstanceQuery) error ListAlertInstances(cmd *models.ListAlertInstancesQuery) error SaveAlertInstance(cmd *models.SaveAlertInstanceCommand) error - FetchOrgIds(cmd *models.FetchUniqueOrgIdsQuery) error + FetchOrgIds() ([]int64, error) } // GetAlertInstance is a handler for retrieving an alert instance based on OrgId, AlertDefintionID, and @@ -24,8 +24,8 @@ func (st DBstore) GetAlertInstance(cmd *models.GetAlertInstanceQuery) error { s := strings.Builder{} s.WriteString(`SELECT * FROM alert_instance WHERE - def_org_id=? AND - def_uid=? AND + rule_org_id=? AND + rule_uid=? AND labels_hash=? `) @@ -38,7 +38,7 @@ func (st DBstore) GetAlertInstance(cmd *models.GetAlertInstanceQuery) error { has, err := sess.SQL(s.String(), params...).Get(&instance) if !has { - return fmt.Errorf("instance not found for labels %v (hash: %v), alert definition %v (org %v)", cmd.Labels, hash, cmd.RuleUID, cmd.RuleOrgID) + return fmt.Errorf("instance not found for labels %v (hash: %v), alert rule %v (org %v)", cmd.Labels, hash, cmd.RuleUID, cmd.RuleOrgID) } if err != nil { return err @@ -63,10 +63,10 @@ func (st DBstore) ListAlertInstances(cmd *models.ListAlertInstancesQuery) error params = append(params, p...) } - addToQuery("SELECT alert_instance.*, alert_definition.title AS def_title FROM alert_instance LEFT JOIN alert_definition ON alert_instance.def_org_id = alert_definition.org_id AND alert_instance.def_uid = alert_definition.uid WHERE def_org_id = ?", cmd.RuleOrgID) + addToQuery("SELECT alert_instance.*, alert_rule.title AS rule_title FROM alert_instance LEFT JOIN alert_rule ON alert_instance.rule_org_id = alert_rule.org_id AND alert_instance.rule_uid = alert_rule.uid WHERE rule_org_id = ?", cmd.RuleOrgID) if cmd.RuleUID != "" { - addToQuery(` AND def_uid = ?`, cmd.RuleUID) + addToQuery(` AND rule_uid = ?`, cmd.RuleUID) } if cmd.State != "" { @@ -109,8 +109,8 @@ func (st DBstore) SaveAlertInstance(cmd *models.SaveAlertInstanceCommand) error upsertSQL := st.SQLStore.Dialect.UpsertSQL( "alert_instance", - []string{"def_org_id", "def_uid", "labels_hash"}, - []string{"def_org_id", "def_uid", "labels", "labels_hash", "current_state", "current_state_since", "current_state_end", "last_eval_time"}) + []string{"rule_org_id", "rule_uid", "labels_hash"}, + []string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_state_since", "current_state_end", "last_eval_time"}) _, err = sess.SQL(upsertSQL, params...).Query() if err != nil { return err @@ -120,10 +120,10 @@ func (st DBstore) SaveAlertInstance(cmd *models.SaveAlertInstanceCommand) error }) } -func (st DBstore) FetchOrgIds(cmd *models.FetchUniqueOrgIdsQuery) error { - return st.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { - orgIds := make([]*models.FetchUniqueOrgIdsQueryResult, 0) +func (st DBstore) FetchOrgIds() ([]int64, error) { + orgIds := []int64{} + err := st.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { s := strings.Builder{} params := make([]interface{}, 0) @@ -132,13 +132,13 @@ func (st DBstore) FetchOrgIds(cmd *models.FetchUniqueOrgIdsQuery) error { params = append(params, p...) } - addToQuery("SELECT DISTINCT def_org_id FROM alert_instance") + addToQuery("SELECT DISTINCT rule_org_id FROM alert_instance") if err := sess.SQL(s.String(), params...).Find(&orgIds); err != nil { return err } - - cmd.Result = orgIds return nil }) + + return orgIds, err } diff --git a/pkg/services/ngalert/tests/instance_database_test.go b/pkg/services/ngalert/tests/instance_database_test.go index 79c5532cc5f..95aba4c14a9 100644 --- a/pkg/services/ngalert/tests/instance_database_test.go +++ b/pkg/services/ngalert/tests/instance_database_test.go @@ -172,7 +172,7 @@ func TestAlertInstanceOperations(t *testing.T) { require.Len(t, listQuery.Result, 1) require.Equal(t, saveCmdTwo.RuleOrgID, listQuery.Result[0].RuleOrgID) - require.Equal(t, saveCmdTwo.RuleUID, listQuery.Result[0].RuleDefinitionUID) + require.Equal(t, saveCmdTwo.RuleUID, listQuery.Result[0].RuleUID) require.Equal(t, saveCmdTwo.Labels, listQuery.Result[0].Labels) require.Equal(t, saveCmdTwo.State, listQuery.Result[0].CurrentState) }) diff --git a/pkg/services/schemaloader/schemaloader.go b/pkg/services/schemaloader/schemaloader.go index 6ea729778c2..647c6b206eb 100644 --- a/pkg/services/schemaloader/schemaloader.go +++ b/pkg/services/schemaloader/schemaloader.go @@ -62,7 +62,7 @@ func (rs *SchemaLoaderService) DashboardApplyDefaults(input *simplejson.Json) (* val = removeNils(val) data, _ := json.Marshal(val) dsSchema := schema.Find(rs.DashFamily, schema.Latest()) - result, err := dsSchema.ApplyDefaults(schema.Resource{Value: data}) + result, err := schema.ApplyDefaults(schema.Resource{Value: data}, dsSchema.CUE()) if err != nil { return input, err } @@ -83,7 +83,7 @@ func (rs *SchemaLoaderService) DashboardTrimDefaults(input simplejson.Json) (sim return input, err } // spew.Dump(dsSchema) - result, err := dsSchema.TrimDefaults(schema.Resource{Value: data}) + result, err := schema.TrimDefaults(schema.Resource{Value: data}, dsSchema.CUE()) if err != nil { return input, err } diff --git a/pkg/services/sqlstore/annotation_cleanup.go b/pkg/services/sqlstore/annotation_cleanup.go index 78227366530..0f9a181777b 100644 --- a/pkg/services/sqlstore/annotation_cleanup.go +++ b/pkg/services/sqlstore/annotation_cleanup.go @@ -47,8 +47,9 @@ func (acs *AnnotationCleanupService) CleanAnnotations(ctx context.Context, cfg * if err != nil { return totalCleanedAnnotations, 0, err } - - affected, err = acs.cleanOrphanedAnnotationTags(ctx) + if totalCleanedAnnotations > 0 { + affected, err = acs.cleanOrphanedAnnotationTags(ctx) + } return totalCleanedAnnotations, affected, err } diff --git a/pkg/services/sqlstore/org_test.go b/pkg/services/sqlstore/org_test.go index 37a93ed7172..4f089b0fe8d 100644 --- a/pkg/services/sqlstore/org_test.go +++ b/pkg/services/sqlstore/org_test.go @@ -95,6 +95,43 @@ func TestAccountDataAccess(t *testing.T) { }) }) + Convey("Given single org and 2 users inserted", func() { + setting.AutoAssignOrg = true + setting.AutoAssignOrgId = 1 + setting.AutoAssignOrgRole = "Viewer" + + ac1cmd := models.CreateUserCommand{Login: "ac1", Email: "ac1@test.com", Name: "ac1 name"} + ac2cmd := models.CreateUserCommand{Login: "ac2", Email: "ac2@test.com", Name: "ac2 name"} + + ac1, err := sqlStore.CreateUser(context.Background(), ac1cmd) + So(err, ShouldBeNil) + _, err = sqlStore.CreateUser(context.Background(), ac2cmd) + So(err, ShouldBeNil) + + Convey("Can get organization users paginated with query", func() { + query := models.SearchOrgUsersQuery{ + OrgID: ac1.OrgId, + Page: 1, + } + err = sqlStore.SearchOrgUsers(&query) + + So(err, ShouldBeNil) + So(len(query.Result.OrgUsers), ShouldEqual, 2) + }) + + Convey("Can get organization users paginated and limited", func() { + query := models.SearchOrgUsersQuery{ + OrgID: ac1.OrgId, + Limit: 1, + Page: 1, + } + err = sqlStore.SearchOrgUsers(&query) + + So(err, ShouldBeNil) + So(len(query.Result.OrgUsers), ShouldEqual, 1) + }) + }) + Convey("Given two saved users", func() { setting.AutoAssignOrg = false diff --git a/pkg/services/sqlstore/org_users.go b/pkg/services/sqlstore/org_users.go index 320e25a584b..7105634c42d 100644 --- a/pkg/services/sqlstore/org_users.go +++ b/pkg/services/sqlstore/org_users.go @@ -142,6 +142,71 @@ func GetOrgUsers(query *models.GetOrgUsersQuery) error { return nil } +func (ss *SQLStore) SearchOrgUsers(query *models.SearchOrgUsersQuery) error { + query.Result = models.SearchOrgUsersQueryResult{ + OrgUsers: make([]*models.OrgUserDTO, 0), + } + + sess := x.Table("org_user") + sess.Join("INNER", x.Dialect().Quote("user"), fmt.Sprintf("org_user.user_id=%s.id", x.Dialect().Quote("user"))) + + whereConditions := make([]string, 0) + whereParams := make([]interface{}, 0) + + whereConditions = append(whereConditions, "org_user.org_id = ?") + whereParams = append(whereParams, query.OrgID) + + if query.Query != "" { + queryWithWildcards := "%" + query.Query + "%" + whereConditions = append(whereConditions, "(email "+dialect.LikeStr()+" ? OR name "+dialect.LikeStr()+" ? OR login "+dialect.LikeStr()+" ?)") + whereParams = append(whereParams, queryWithWildcards, queryWithWildcards, queryWithWildcards) + } + + if len(whereConditions) > 0 { + sess.Where(strings.Join(whereConditions, " AND "), whereParams...) + } + + if query.Limit > 0 { + offset := query.Limit * (query.Page - 1) + sess.Limit(query.Limit, offset) + } + + sess.Cols( + "org_user.org_id", + "org_user.user_id", + "user.email", + "user.name", + "user.login", + "org_user.role", + "user.last_seen_at", + ) + sess.Asc("user.email", "user.login") + + if err := sess.Find(&query.Result.OrgUsers); err != nil { + return err + } + + // get total count + orgUser := models.OrgUser{} + countSess := x.Table("org_user") + + if len(whereConditions) > 0 { + countSess.Where(strings.Join(whereConditions, " AND "), whereParams...) + } + + count, err := countSess.Count(&orgUser) + if err != nil { + return err + } + query.Result.TotalCount = count + + for _, user := range query.Result.OrgUsers { + user.LastSeenAtAge = util.GetAgeString(user.LastSeenAt) + } + + return nil +} + func RemoveOrgUser(cmd *models.RemoveOrgUserCommand) error { return inTransaction(func(sess *DBSession) error { // check if user exists diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go index 6e9cefd8641..946380f028d 100644 --- a/pkg/tsdb/loki/loki.go +++ b/pkg/tsdb/loki/loki.go @@ -3,6 +3,7 @@ package loki import ( "context" "fmt" + "net/http" "regexp" "strings" "time" @@ -53,6 +54,11 @@ func (e *LokiExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource, return plugins.DataResponse{}, err } + transport, err := dsInfo.GetHttpTransport() + if err != nil { + return plugins.DataResponse{}, err + } + client := &client.DefaultClient{ Address: dsInfo.Url, Username: dsInfo.BasicAuthUser, @@ -60,6 +66,9 @@ func (e *LokiExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource, TLSConfig: config.TLSConfig{ InsecureSkipVerify: tlsConfig.InsecureSkipVerify, }, + Tripperware: func(t http.RoundTripper) http.RoundTripper { + return transport + }, } queries, err := e.parseQuery(dsInfo, queryContext) diff --git a/public/app/angular/AngularLocationWrapper.test.ts b/public/app/angular/AngularLocationWrapper.test.ts index 3b4d0d2b8d8..55b3421a036 100644 --- a/public/app/angular/AngularLocationWrapper.test.ts +++ b/public/app/angular/AngularLocationWrapper.test.ts @@ -85,19 +85,19 @@ describe('AngularLocationWrapper', () => { it('search() should accept object', function () { locationService.push('/path/b'); - wrapper.search({ one: 1, two: true }); - expect(wrapper.search()).toEqual({ one: 1, two: true }); + wrapper.search({ one: '1', two: true }); + expect(wrapper.search()).toEqual({ one: '1', two: true }); expect(wrapper.absUrl()).toBe('http://www.domain.com:9877/path/b?one=1&two'); }); it('should copy object', function () { locationService.push('/path/b'); - const obj: Record = { one: 1, two: true, three: null }; + const obj: Record = { one: '1', two: true, three: null }; wrapper.search(obj); - expect(obj).toEqual({ one: 1, two: true, three: null }); + expect(obj).toEqual({ one: '1', two: true, three: null }); obj.one = 'changed'; - expect(wrapper.search()).toEqual({ one: 1, two: true }); + expect(wrapper.search()).toEqual({ one: '1', two: true }); expect(wrapper.absUrl()).toBe('http://www.domain.com:9877/path/b?one=1&two'); }); @@ -117,8 +117,8 @@ describe('AngularLocationWrapper', () => { it('should remove multiple parameters', function () { locationService.push('/path/b'); - wrapper.search({ one: 1, two: true }); - expect(wrapper.search()).toEqual({ one: 1, two: true }); + wrapper.search({ one: '1', two: true }); + expect(wrapper.search()).toEqual({ one: '1', two: true }); wrapper.search({ one: null, two: null }); expect(wrapper.search()).toEqual({}); diff --git a/public/app/core/components/Select/OrgPicker.tsx b/public/app/core/components/Select/OrgPicker.tsx index e5a0433c754..77299295d62 100644 --- a/public/app/core/components/Select/OrgPicker.tsx +++ b/public/app/core/components/Select/OrgPicker.tsx @@ -36,7 +36,7 @@ export class OrgPicker extends PureComponent { } getOrgOptions = async (query: string): Promise>> => { - if (!this.orgs) { + if (!this.orgs?.length) { await this.loadOrgs(); } return this.orgs.map( diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 6a7da4df2fc..79c5d7d9755 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -30,6 +30,7 @@ import { AbsoluteTimeRange, sortInAscendingOrder, rangeUtil, + DataQuery, } from '@grafana/data'; import { getThemeColor } from 'app/core/utils/colors'; import { config } from '@grafana/runtime'; @@ -202,7 +203,8 @@ export function dataFrameToLogsModel( dataFrame: DataFrame[], intervalMs: number | undefined, timeZone: TimeZone, - absoluteRange?: AbsoluteTimeRange + absoluteRange?: AbsoluteTimeRange, + queries?: DataQuery[] ): LogsModel { const { logSeries } = separateLogsAndMetrics(dataFrame); const logsModel = logSeriesToLogsModel(logSeries); @@ -225,6 +227,7 @@ export function dataFrameToLogsModel( } else { logsModel.series = []; } + logsModel.queries = queries; return logsModel; } @@ -233,6 +236,7 @@ export function dataFrameToLogsModel( rows: [], meta: [], series: [], + queries, }; } diff --git a/public/app/features/admin/UserOrgs.tsx b/public/app/features/admin/UserOrgs.tsx index 85232d0ee54..083650ace92 100644 --- a/public/app/features/admin/UserOrgs.tsx +++ b/public/app/features/admin/UserOrgs.tsx @@ -1,16 +1,6 @@ import React, { PureComponent } from 'react'; import { css, cx } from '@emotion/css'; -import { - Button, - ConfirmButton, - Container, - Field, - HorizontalGroup, - Modal, - stylesFactory, - Themeable, - withTheme, -} from '@grafana/ui'; +import { Button, ConfirmButton, Field, HorizontalGroup, Modal, stylesFactory, Themeable, withTheme } from '@grafana/ui'; import { GrafanaTheme } from '@grafana/data'; import { AccessControlAction, Organization, OrgRole, UserOrg } from 'app/types'; import { OrgPicker, OrgSelectItem } from 'app/core/components/Select/OrgPicker'; @@ -251,16 +241,16 @@ export class AddToOrgModal extends PureComponent - + + - - + ); } diff --git a/public/app/features/dashboard/components/PanelEditor/state/actions.ts b/public/app/features/dashboard/components/PanelEditor/state/actions.ts index bde2d105533..b0b23b6681a 100644 --- a/public/app/features/dashboard/components/PanelEditor/state/actions.ts +++ b/public/app/features/dashboard/components/PanelEditor/state/actions.ts @@ -104,9 +104,11 @@ function updateDuplicateLibraryPanels(modifiedPanel: PanelModel, dashboard: Dash // Loaded plugin is not included in the persisted properties // So is not handled by restoreModel - panel.plugin = modifiedSaveModel.plugin; + const pluginChanged = panel.plugin?.meta.id !== modifiedPanel.plugin?.meta.id; + panel.plugin = modifiedPanel.plugin; + panel.configRev++; - if (panel.type !== modifiedPanel.type) { + if (pluginChanged) { dispatch(panelModelAndPluginReady({ panelId: panel.id, plugin: panel.plugin! })); } diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index d921ad91ef0..0b538c1ff0d 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -47,6 +47,7 @@ interface Props { logRows: LogRowModel[]; logsMeta?: LogsMetaItem[]; logsSeries?: GraphSeriesXY[]; + logsQueries?: DataQuery[]; visibleRange?: AbsoluteTimeRange; width: number; theme: GrafanaTheme; @@ -56,7 +57,6 @@ interface Props { timeZone: TimeZone; scanning?: boolean; scanRange?: RawTimeRange; - queries: DataQuery[]; showContextToggle?: (row?: LogRowModel) => boolean; onChangeTime: (range: AbsoluteTimeRange) => void; onClickFilterLabel?: (key: string, value: string) => void; @@ -243,7 +243,7 @@ export class UnthemedLogs extends PureComponent { onChangeTime, getFieldLinks, theme, - queries, + logsQueries, } = this.props; const { @@ -353,12 +353,12 @@ export class UnthemedLogs extends PureComponent {
diff --git a/public/app/features/explore/LogsContainer.tsx b/public/app/features/explore/LogsContainer.tsx index f6cc9ac2f50..47bcfdfe216 100644 --- a/public/app/features/explore/LogsContainer.tsx +++ b/public/app/features/explore/LogsContainer.tsx @@ -64,6 +64,7 @@ export class LogsContainer extends PureComponent @@ -156,7 +156,6 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string } isPaused, range, absoluteRange, - queries, } = item; const timeZone = getTimeZone(state.user); @@ -166,6 +165,7 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string } logRows: logsResult?.rows, logsMeta: logsResult?.meta, logsSeries: logsResult?.series, + logsQueries: logsResult?.queries, visibleRange: logsResult?.visibleRange, scanning, timeZone, @@ -174,7 +174,6 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string } isPaused, range, absoluteRange, - queries, }; } diff --git a/public/app/features/explore/LogsNavigation.tsx b/public/app/features/explore/LogsNavigation.tsx index 9b4ae51dc24..98772edfe98 100644 --- a/public/app/features/explore/LogsNavigation.tsx +++ b/public/app/features/explore/LogsNavigation.tsx @@ -10,7 +10,7 @@ type Props = { timeZone: TimeZone; queries: DataQuery[]; loading: boolean; - visibleRange?: AbsoluteTimeRange; + visibleRange: AbsoluteTimeRange; logsSortOrder?: LogsSortOrder | null; onChangeTime: (range: AbsoluteTimeRange) => void; scrollToTopLogs: () => void; @@ -28,8 +28,8 @@ function LogsNavigation({ loading, onChangeTime, scrollToTopLogs, - visibleRange = absoluteRange, - queries = [], + visibleRange, + queries, }: Props) { const [pages, setPages] = useState([]); const [currentPageIndex, setCurrentPageIndex] = useState(0); @@ -98,9 +98,10 @@ function LogsNavigation({ from: pages[currentPageIndex + 1].queryRange.from, to: pages[currentPageIndex + 1].queryRange.to, }); + } else { + //If we are on the last page, create new range + changeTime({ from: visibleRange.from - rangeSpanRef.current, to: visibleRange.from }); } - //If we are on the last page, create new range - changeTime({ from: visibleRange.from - rangeSpanRef.current, to: visibleRange.from }); }} disabled={loading} > diff --git a/public/app/features/explore/Wrapper.test.tsx b/public/app/features/explore/Wrapper.test.tsx index 5f62c7f90aa..9e58f245200 100644 --- a/public/app/features/explore/Wrapper.test.tsx +++ b/public/app/features/explore/Wrapper.test.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import Wrapper from './Wrapper'; import { configureStore } from '../../store/configureStore'; import { Provider } from 'react-redux'; @@ -52,7 +52,7 @@ describe('Wrapper', () => { // At this point url should be initialised to some defaults expect(locationService.getSearchObject()).toEqual({ - orgId: 1, + orgId: '1', left: JSON.stringify(['now-1h', 'now', 'loki', {}]), }); expect(datasources.loki.query).not.toBeCalled(); @@ -74,7 +74,7 @@ describe('Wrapper', () => { // We did not change the url expect(locationService.getSearchObject()).toEqual({ - orgId: 1, + orgId: '1', ...query, }); @@ -141,7 +141,7 @@ describe('Wrapper', () => { await screen.findByText('elastic Editor input:'); expect(datasources.elastic.query).not.toBeCalled(); expect(locationService.getSearchObject()).toEqual({ - orgId: 1, + orgId: '1', left: JSON.stringify(['now-1h', 'now', 'elastic', {}]), }); }); @@ -183,7 +183,7 @@ describe('Wrapper', () => { // We did not change the url expect(locationService.getSearchObject()).toEqual({ - orgId: 1, + orgId: '1', ...query, }); diff --git a/public/app/features/explore/state/query.ts b/public/app/features/explore/state/query.ts index 4e3785ce37e..c7a38911f17 100644 --- a/public/app/features/explore/state/query.ts +++ b/public/app/features/explore/state/query.ts @@ -357,7 +357,7 @@ export const runQueries = (exploreId: ExploreId, options?: { replaceUrl?: boolea map((data: PanelData) => preProcessPanelData(data, queryResponse)), map(decorateWithFrameTypeMetadata), map(decorateWithGraphResult), - map(decorateWithLogsResult({ absoluteRange, refreshInterval })), + map(decorateWithLogsResult({ absoluteRange, refreshInterval, queries })), mergeMap(decorateWithTableResult) ) .subscribe( diff --git a/public/app/features/explore/utils/decorators.ts b/public/app/features/explore/utils/decorators.ts index cce53862930..dfd2fd4f135 100644 --- a/public/app/features/explore/utils/decorators.ts +++ b/public/app/features/explore/utils/decorators.ts @@ -6,6 +6,7 @@ import { PanelData, sortLogsResult, standardTransformers, + DataQuery, } from '@grafana/data'; import { config } from '@grafana/runtime'; import { groupBy } from 'lodash'; @@ -129,7 +130,7 @@ export const decorateWithTableResult = (data: ExplorePanelData): Observable (data: ExplorePanelData): ExplorePanelData => { if (data.logsFrames.length === 0) { return { ...data, logsResult: null }; @@ -137,7 +138,13 @@ export const decorateWithLogsResult = ( const timeZone = data.request?.timezone ?? 'browser'; const intervalMs = data.request?.intervalMs; - const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, options.absoluteRange); + const newResults = dataFrameToLogsModel( + data.logsFrames, + intervalMs, + timeZone, + options.absoluteRange, + options.queries + ); const sortOrder = refreshIntervalToSortOrder(options.refreshInterval); const sortedNewResults = sortLogsResult(newResults, sortOrder); const rows = sortedNewResults.rows; diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/query_ctrl.ts b/public/app/plugins/datasource/grafana-azure-monitor-datasource/query_ctrl.ts index d5f7e21cc02..d2348f6e93b 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/query_ctrl.ts +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/query_ctrl.ts @@ -33,8 +33,8 @@ export class AzureMonitorQueryCtrl extends QueryCtrl { reactQueryEditors = [ AzureQueryType.AzureMonitor, AzureQueryType.LogAnalytics, - AzureQueryType.ApplicationInsights, - AzureQueryType.InsightsAnalytics, + // AzureQueryType.ApplicationInsights, + // AzureQueryType.InsightsAnalytics, ]; // target: AzureMonitorQuery; diff --git a/public/app/plugins/datasource/loki/components/LokiLabelBrowser.tsx b/public/app/plugins/datasource/loki/components/LokiLabelBrowser.tsx index 3f03e8136a3..80ffc5c8ec2 100644 --- a/public/app/plugins/datasource/loki/components/LokiLabelBrowser.tsx +++ b/public/app/plugins/datasource/loki/components/LokiLabelBrowser.tsx @@ -1,6 +1,7 @@ import React, { ChangeEvent } from 'react'; import { Button, HorizontalGroup, Input, Label, LoadingPlaceholder, stylesFactory, withTheme } from '@grafana/ui'; import LokiLanguageProvider from '../language_provider'; +import PromQlLanguageProvider from '../../prometheus/language_provider'; import { css, cx } from '@emotion/css'; import store from 'app/core/store'; import { FixedSizeList } from 'react-window'; @@ -16,7 +17,8 @@ const EMPTY_SELECTOR = '{}'; export const LAST_USED_LABELS_KEY = 'grafana.datasources.loki.browser.labels'; export interface BrowserProps { - languageProvider: LokiLanguageProvider; + // TODO #33976: Is it possible to use a common interface here? For example: LabelsLanguageProvider + languageProvider: LokiLanguageProvider | PromQlLanguageProvider; onChange: (selector: string) => void; theme: GrafanaTheme; autoSelect?: number; @@ -333,7 +335,7 @@ export class UnthemedLokiLabelBrowser extends React.Component { + onChangeLabelBrowser = (selector: string) => { this.onChangeQuery(selector, true); this.setState({ labelBrowserVisible: false }); }; @@ -174,7 +174,7 @@ export class LokiQueryField extends React.PureComponent {labelBrowserVisible && (
- +
)} diff --git a/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap b/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap index 5273e5490e0..a192fdc9e40 100644 --- a/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap +++ b/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap @@ -68,6 +68,7 @@ exports[`LokiExploreQueryEditor should render component 1`] = ` "getBeginningCompletionItems": [Function], "getPipeCompletionItem": [Function], "getTermCompletionItems": [Function], + "labelFetchTs": 0, "labelKeys": Array [], "labelsCache": LRUCache { Symbol(max): 10, @@ -85,7 +86,6 @@ exports[`LokiExploreQueryEditor should render component 1`] = ` }, Symbol(length): 0, }, - "logLabelFetchTs": 0, "lookupsDisabled": false, "request": [Function], "seriesCache": LRUCache { diff --git a/public/app/plugins/datasource/loki/language_provider.test.ts b/public/app/plugins/datasource/loki/language_provider.test.ts index 9d8476903f6..9d162ac7db0 100644 --- a/public/app/plugins/datasource/loki/language_provider.test.ts +++ b/public/app/plugins/datasource/loki/language_provider.test.ts @@ -221,7 +221,7 @@ describe('Request URL', () => { const datasourceSpy = jest.spyOn(datasourceWithLabels as any, 'metadataRequest'); const instance = new LanguageProvider(datasourceWithLabels); - instance.fetchLogLabels(); + instance.fetchLabels(); const expectedUrl = '/loki/api/v1/label'; expect(datasourceSpy).toHaveBeenCalledWith(expectedUrl, rangeParams); }); diff --git a/public/app/plugins/datasource/loki/language_provider.ts b/public/app/plugins/datasource/loki/language_provider.ts index 012b2c5b9e9..9df235d0bdf 100644 --- a/public/app/plugins/datasource/loki/language_provider.ts +++ b/public/app/plugins/datasource/loki/language_provider.ts @@ -70,7 +70,7 @@ export function addHistoryMetadata(item: CompletionItem, history: LokiHistoryIte export default class LokiLanguageProvider extends LanguageProvider { labelKeys: string[]; - logLabelFetchTs: number; + labelFetchTs: number; started = false; datasource: LokiDatasource; lookupsDisabled = false; // Dynamically set to true for big/slow instances @@ -88,7 +88,7 @@ export default class LokiLanguageProvider extends LanguageProvider { this.datasource = datasource; this.labelKeys = []; - this.logLabelFetchTs = 0; + this.labelFetchTs = 0; Object.assign(this, initialValues); } @@ -116,7 +116,7 @@ export default class LokiLanguageProvider extends LanguageProvider { */ start = () => { if (!this.startTask) { - this.startTask = this.fetchLogLabels().then(() => { + this.startTask = this.fetchLabels().then(() => { this.started = true; return []; }); @@ -415,12 +415,11 @@ export default class LokiLanguageProvider extends LanguageProvider { /** * Fetches all label keys - * @param absoluteRange Fetches */ - async fetchLogLabels(): Promise { + async fetchLabels(): Promise { const url = '/loki/api/v1/label'; const timeRange = this.datasource.getTimeRangeParams(); - this.logLabelFetchTs = Date.now().valueOf(); + this.labelFetchTs = Date.now().valueOf(); const res = await this.request(url, timeRange); if (Array.isArray(res)) { @@ -431,8 +430,8 @@ export default class LokiLanguageProvider extends LanguageProvider { } async refreshLogLabels(forceRefresh?: boolean) { - if ((this.labelKeys && Date.now().valueOf() - this.logLabelFetchTs > LABEL_REFRESH_INTERVAL) || forceRefresh) { - await this.fetchLogLabels(); + if ((this.labelKeys && Date.now().valueOf() - this.labelFetchTs > LABEL_REFRESH_INTERVAL) || forceRefresh) { + await this.fetchLabels(); } } @@ -495,17 +494,17 @@ export default class LokiLanguageProvider extends LanguageProvider { const cacheKey = this.generateCacheKey(url, start, end, key); const params = { start, end }; - let labelValue = this.labelsCache.get(cacheKey); - if (!labelValue) { + let labelValues = this.labelsCache.get(cacheKey); + if (!labelValues) { // Clear value when requesting new one. Empty object being truthy also makes sure we don't request twice. this.labelsCache.set(cacheKey, []); const res = await this.request(url, params); if (Array.isArray(res)) { - labelValue = res.slice().sort(); - this.labelsCache.set(cacheKey, labelValue); + labelValues = res.slice().sort(); + this.labelsCache.set(cacheKey, labelValues); } } - return labelValue ?? []; + return labelValues ?? []; } } diff --git a/public/app/plugins/datasource/prometheus/components/Label.tsx b/public/app/plugins/datasource/prometheus/components/Label.tsx new file mode 100644 index 00000000000..43831b23fc6 --- /dev/null +++ b/public/app/plugins/datasource/prometheus/components/Label.tsx @@ -0,0 +1,123 @@ +import React, { forwardRef, HTMLAttributes } from 'react'; +import { cx, css } from '@emotion/css'; +import { GrafanaTheme2 } from '@grafana/data'; +import { useTheme2 } from '@grafana/ui'; +// @ts-ignore +import Highlighter from 'react-highlight-words'; + +/** + * @public + */ +export type OnLabelClick = (name: string, value: string | undefined, event: React.MouseEvent) => void; + +export interface Props extends Omit, 'onClick'> { + name: string; + active?: boolean; + loading?: boolean; + searchTerm?: string; + value?: string; + facets?: number; + onClick?: OnLabelClick; +} + +/** + * TODO #33976: Create a common, shared component with public/app/plugins/datasource/loki/components/LokiLabel.tsx + */ +export const Label = forwardRef( + ({ name, value, hidden, facets, onClick, className, loading, searchTerm, active, style, ...rest }, ref) => { + const theme = useTheme2(); + const styles = getLabelStyles(theme); + const searchWords = searchTerm ? [searchTerm] : []; + + const onLabelClick = (event: React.MouseEvent) => { + if (onClick && !hidden) { + onClick(name, value, event); + } + }; + // Using this component for labels and label values. If value is given use value for display text. + let text = value || name; + if (facets) { + text = `${text} (${facets})`; + } + + return ( + + ); + } +); + +Label.displayName = 'Label'; + +const getLabelStyles = (theme: GrafanaTheme2) => ({ + base: css` + cursor: pointer; + font-size: ${theme.typography.size.sm}; + line-height: ${theme.typography.bodySmall.lineHeight}; + background-color: ${theme.colors.background.secondary}; + vertical-align: baseline; + color: ${theme.colors.text}; + white-space: nowrap; + text-shadow: none; + padding: ${theme.spacing(0.5)}; + border-radius: ${theme.shape.borderRadius()}; + margin-right: ${theme.spacing(1)}; + margin-bottom: ${theme.spacing(0.5)}; + `, + loading: css` + font-weight: ${theme.typography.fontWeightMedium}; + background-color: ${theme.colors.primary.shade}; + color: ${theme.colors.text.primary}; + animation: pulse 3s ease-out 0s infinite normal forwards; + @keyframes pulse { + 0% { + color: ${theme.colors.text.primary}; + } + 50% { + color: ${theme.colors.text.secondary}; + } + 100% { + color: ${theme.colors.text.disabled}; + } + } + `, + active: css` + font-weight: ${theme.typography.fontWeightMedium}; + background-color: ${theme.colors.primary.main}; + color: ${theme.colors.primary.contrastText}; + `, + matchHighLight: css` + background: inherit; + color: ${theme.colors.primary.text}; + background-color: ${theme.colors.primary.transparent}; + `, + hidden: css` + opacity: 0.6; + cursor: default; + border: 1px solid transparent; + `, + hover: css` + &:hover { + opacity: 0.85; + cursor: pointer; + } + `, +}); diff --git a/public/app/plugins/datasource/prometheus/components/PromExploreQueryEditor.test.tsx b/public/app/plugins/datasource/prometheus/components/PromExploreQueryEditor.test.tsx index 6501fe7f9a1..ea45b588f86 100644 --- a/public/app/plugins/datasource/prometheus/components/PromExploreQueryEditor.test.tsx +++ b/public/app/plugins/datasource/prometheus/components/PromExploreQueryEditor.test.tsx @@ -10,6 +10,8 @@ const setup = (renderMethod: any, propOverrides?: object) => { const datasourceMock: unknown = { languageProvider: { syntax: () => {}, + getLabelKeys: () => [], + metrics: [], }, }; const datasource: PrometheusDatasource = datasourceMock as PrometheusDatasource; diff --git a/public/app/plugins/datasource/prometheus/components/PromQueryField.test.tsx b/public/app/plugins/datasource/prometheus/components/PromQueryField.test.tsx index b50a44564f9..f05bf5712d3 100644 --- a/public/app/plugins/datasource/prometheus/components/PromQueryField.test.tsx +++ b/public/app/plugins/datasource/prometheus/components/PromQueryField.test.tsx @@ -2,10 +2,10 @@ import RCCascader from 'rc-cascader'; import React from 'react'; import PromQlLanguageProvider from '../language_provider'; -import PromQueryField, { groupMetricsByPrefix, RECORDING_RULES_GROUP } from './PromQueryField'; -import { DataSourceInstanceSettings, dateTime } from '@grafana/data'; +import PromQueryField from './PromQueryField'; +import { DataSourceInstanceSettings } from '@grafana/data'; import { PromOptions } from '../types'; -import { fireEvent, render, screen } from '@testing-library/react'; +import { render, screen } from '@testing-library/react'; describe('PromQueryField', () => { beforeAll(() => { @@ -18,6 +18,8 @@ describe('PromQueryField', () => { languageProvider: { start: () => Promise.resolve([]), syntax: () => {}, + getLabelKeys: () => [], + metrics: [], }, } as unknown) as DataSourceInstanceSettings; @@ -40,6 +42,8 @@ describe('PromQueryField', () => { languageProvider: { start: () => Promise.resolve([]), syntax: () => {}, + getLabelKeys: () => [], + metrics: [], }, } as unknown) as DataSourceInstanceSettings; const queryField = render( @@ -75,8 +79,6 @@ describe('PromQueryField', () => { /> ); - checkMetricsInCascader(await screen.findByRole('button'), metrics); - const changedMetrics = ['baz', 'moo']; queryField.rerender( { /> ); - // If we check the cascader right away it should be in loading state - let cascader = screen.getByRole('button'); - expect(cascader.textContent).toContain('Loading'); - checkMetricsInCascader(await screen.findByRole('button'), changedMetrics); - }); - - it('does not refreshes metrics when after rounding to minute time range does not change', async () => { - const defaultProps = { - query: { expr: '', refId: '' }, - onRunQuery: () => {}, - onChange: () => {}, - history: [], - }; - const metrics = ['foo', 'bar']; - const changedMetrics = ['foo', 'baz']; - const range = { - from: dateTime('2020-10-28T00:00:00Z'), - to: dateTime('2020-10-28T01:00:00Z'), - }; - - const languageProvider = makeLanguageProvider({ metrics: [metrics, changedMetrics] }); - const queryField = render( - - ); - checkMetricsInCascader(await screen.findByRole('button'), metrics); - - const newRange = { - from: dateTime('2020-10-28T00:00:01Z'), - to: dateTime('2020-10-28T01:00:01Z'), - }; - queryField.rerender( - - ); - let cascader = screen.getByRole('button'); - // Should not show loading - expect(cascader.textContent).toContain('Metrics'); - checkMetricsInCascader(await screen.findByRole('button'), metrics); - }); - - it('refreshes metrics when time range changes but dont show loading state', async () => { - const defaultProps = { - query: { expr: '', refId: '' }, - onRunQuery: () => {}, - onChange: () => {}, - history: [], - }; - const metrics = ['foo', 'bar']; - const changedMetrics = ['baz', 'moo']; - const range = { - from: dateTime('2020-10-28T00:00:00Z'), - to: dateTime('2020-10-28T01:00:00Z'), - }; - - const languageProvider = makeLanguageProvider({ metrics: [metrics, changedMetrics] }); - const queryField = render( - - ); - checkMetricsInCascader(await screen.findByRole('button'), metrics); - - const newRange = { - from: dateTime('2020-10-28T01:00:00Z'), - to: dateTime('2020-10-28T02:00:00Z'), - }; - queryField.rerender( - - ); - let cascader = screen.getByRole('button'); - // Should not show loading - expect(cascader.textContent).toContain('Metrics'); - checkMetricsInCascader(cascader, metrics); - }); -}); - -describe('groupMetricsByPrefix()', () => { - it('returns an empty group for no metrics', () => { - expect(groupMetricsByPrefix([])).toEqual([]); - }); - - it('returns options grouped by prefix', () => { - expect(groupMetricsByPrefix(['foo_metric'])).toMatchObject([ - { - value: 'foo', - children: [ - { - value: 'foo_metric', - }, - ], - }, - ]); - }); - - it('returns options grouped by prefix with metadata', () => { - expect(groupMetricsByPrefix(['foo_metric'], { foo_metric: [{ type: 'TYPE', help: 'my help' }] })).toMatchObject([ - { - value: 'foo', - children: [ - { - value: 'foo_metric', - title: 'foo_metric\nTYPE\nmy help', - }, - ], - }, - ]); - }); - - it('returns options without prefix as toplevel option', () => { - expect(groupMetricsByPrefix(['metric'])).toMatchObject([ - { - value: 'metric', - }, - ]); - }); - - it('returns recording rules grouped separately', () => { - expect(groupMetricsByPrefix([':foo_metric:'])).toMatchObject([ - { - value: RECORDING_RULES_GROUP, - children: [ - { - value: ':foo_metric:', - }, - ], - }, - ]); + // If we check the label browser right away it should be in loading state + let labelBrowser = screen.getByRole('button'); + expect(labelBrowser.textContent).toContain('Loading'); }); }); @@ -254,17 +103,10 @@ function makeLanguageProvider(options: { metrics: string[][] }) { metrics: [], metricsMetadata: {}, lookupsDisabled: false, + getLabelKeys: () => [], start() { this.metrics = metricsStack.shift(); return Promise.resolve([]); }, } as any) as PromQlLanguageProvider; } - -function checkMetricsInCascader(cascader: HTMLElement, metrics: string[]) { - fireEvent.keyDown(cascader, { keyCode: 40 }); - let listNodes = screen.getAllByRole('menuitem'); - for (const node of listNodes) { - expect(metrics).toContain(node.innerHTML); - } -} diff --git a/public/app/plugins/datasource/prometheus/components/PromQueryField.tsx b/public/app/plugins/datasource/prometheus/components/PromQueryField.tsx index ec08fab5f2a..3b4da651b87 100644 --- a/public/app/plugins/datasource/prometheus/components/PromQueryField.tsx +++ b/public/app/plugins/datasource/prometheus/components/PromQueryField.tsx @@ -1,10 +1,7 @@ -import { chain } from 'lodash'; import React, { ReactNode } from 'react'; import { Plugin } from 'slate'; import { - ButtonCascader, - CascaderOption, SlatePrism, TypeaheadInput, TypeaheadOutput, @@ -12,12 +9,13 @@ import { BracesPlugin, DOMUtil, SuggestionsState, + Icon, } from '@grafana/ui'; import { LanguageMap, languages as prismLanguages } from 'prismjs'; // dom also includes Element polyfills -import { PromQuery, PromOptions, PromMetricsMetadata } from '../types'; +import { PromQuery, PromOptions } from '../types'; import { roundMsToMin } from '../language_utils'; import { CancelablePromise, makePromiseCancelable } from 'app/core/utils/CancelablePromise'; import { @@ -29,11 +27,11 @@ import { TimeRange, } from '@grafana/data'; import { PrometheusDatasource } from '../datasource'; +import { PrometheusMetricsBrowser } from './PrometheusMetricsBrowser'; -const HISTOGRAM_GROUP = '__histograms__'; export const RECORDING_RULES_GROUP = '__recording_rules__'; -function getChooserText(metricsLookupDisabled: boolean, hasSyntax: boolean, metrics: string[]) { +function getChooserText(metricsLookupDisabled: boolean, hasSyntax: boolean, hasMetrics: boolean) { if (metricsLookupDisabled) { return '(Disabled)'; } @@ -42,56 +40,11 @@ function getChooserText(metricsLookupDisabled: boolean, hasSyntax: boolean, metr return 'Loading metrics...'; } - if (metrics && metrics.length === 0) { + if (!hasMetrics) { return '(No metrics found)'; } - return 'Metrics'; -} - -function addMetricsMetadata(metric: string, metadata?: PromMetricsMetadata): CascaderOption { - const option: CascaderOption = { label: metric, value: metric }; - if (metadata && metadata[metric]) { - const { type = '', help } = metadata[metric][0]; - option.title = [metric, type.toUpperCase(), help].join('\n'); - } - return option; -} - -export function groupMetricsByPrefix(metrics: string[], metadata?: PromMetricsMetadata): CascaderOption[] { - // Filter out recording rules and insert as first option - const ruleRegex = /:\w+:/; - const ruleNames = metrics.filter((metric) => ruleRegex.test(metric)); - const rulesOption = { - label: 'Recording rules', - value: RECORDING_RULES_GROUP, - children: ruleNames - .slice() - .sort() - .map((name) => ({ label: name, value: name })), - }; - - const options = ruleNames.length > 0 ? [rulesOption] : []; - - const delimiter = '_'; - const metricsOptions = chain(metrics) - .filter((metric: string) => !ruleRegex.test(metric)) - .groupBy((metric: string) => metric.split(delimiter)[0]) - .map( - (metricsForPrefix: string[], prefix: string): CascaderOption => { - const prefixIsMetric = metricsForPrefix.length === 1 && metricsForPrefix[0] === prefix; - const children = prefixIsMetric ? [] : metricsForPrefix.sort().map((m) => addMetricsMetadata(m, metadata)); - return { - children, - label: prefix, - value: prefix, - }; - } - ) - .sortBy('label') - .value(); - - return [...options, ...metricsOptions]; + return 'Metrics browser'; } export function willApplySuggestion(suggestion: string, { typeaheadContext, typeaheadText }: SuggestionsState): string { @@ -127,7 +80,7 @@ interface PromQueryFieldProps extends ExploreQueryFieldProps { - let query; - if (selectedOptions.length === 1) { - const selectedOption = selectedOptions[0]; - if (!selectedOption.children || selectedOption.children.length === 0) { - query = selectedOption.value; - } else { - // Ignore click on group - return; - } - } else { - const prefix = selectedOptions[0].value; - const metric = selectedOptions[1].value; - if (prefix === HISTOGRAM_GROUP) { - query = `histogram_quantile(0.95, sum(rate(${metric}[5m])) by (le))`; - } else { - query = metric; - } - } - this.onChangeQuery(query, true); + /** + * TODO #33976: Remove this, add histogram group (query = `histogram_quantile(0.95, sum(rate(${metric}[5m])) by (le))`;) + */ + onChangeLabelBrowser = (selector: string) => { + this.onChangeQuery(selector, true); + this.setState({ labelBrowserVisible: false }); }; onChangeQuery = (value: string, override?: boolean) => { @@ -282,6 +220,10 @@ class PromQueryField extends React.PureComponent { + this.setState((state) => ({ labelBrowserVisible: !state.labelBrowserVisible })); + }; + onClickHintFix = () => { const { datasource, query, onChange, onRunQuery } = this.props; const { hint } = this.state; @@ -294,24 +236,13 @@ class PromQueryField extends React.PureComponent ({ label: hm, value: hm })); - const metricsOptions = - histogramMetrics.length > 0 - ? [ - { label: 'Histograms', value: HISTOGRAM_GROUP, children: histogramOptions, isLeaf: false }, - ...metricsByPrefix, - ] - : metricsByPrefix; - - this.setState({ metricsOptions, syntaxLoaded: true }); + this.setState({ syntaxLoaded: true }); }; onTypeahead = async (typeahead: TypeaheadInput): Promise => { @@ -341,19 +272,24 @@ class PromQueryField extends React.PureComponent 0); + const hasMetrics = languageProvider.metrics.length > 0; + const chooserText = getChooserText(datasource.lookupsDisabled, syntaxLoaded, hasMetrics); + const buttonDisabled = !(syntaxLoaded && hasMetrics); return ( <>
-
- - {chooserText} - -
+ +
+ {labelBrowserVisible && ( +
+ +
+ )} + {ExtraFieldElement} {hint ? (
diff --git a/public/app/plugins/datasource/prometheus/components/PrometheusMetricsBrowser.test.tsx b/public/app/plugins/datasource/prometheus/components/PrometheusMetricsBrowser.test.tsx new file mode 100644 index 00000000000..bb842463fa0 --- /dev/null +++ b/public/app/plugins/datasource/prometheus/components/PrometheusMetricsBrowser.test.tsx @@ -0,0 +1,265 @@ +import React from 'react'; +import { render, screen, waitFor, waitForElementToBeRemoved } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import { getTheme } from '@grafana/ui'; +import { + buildSelector, + facetLabels, + SelectableLabel, + UnthemedPrometheusMetricsBrowser, + BrowserProps, +} from './PrometheusMetricsBrowser'; +import PromQlLanguageProvider from '../language_provider'; + +describe('buildSelector()', () => { + it('returns an empty selector for no labels', () => { + expect(buildSelector([])).toEqual('{}'); + }); + it('returns an empty selector for selected labels with no values', () => { + const labels: SelectableLabel[] = [{ name: 'foo', selected: true }]; + expect(buildSelector(labels)).toEqual('{}'); + }); + it('returns an empty selector for one selected label with no selected values', () => { + const labels: SelectableLabel[] = [{ name: 'foo', selected: true, values: [{ name: 'bar' }] }]; + expect(buildSelector(labels)).toEqual('{}'); + }); + it('returns a simple selector from a selected label with a selected value', () => { + const labels: SelectableLabel[] = [{ name: 'foo', selected: true, values: [{ name: 'bar', selected: true }] }]; + expect(buildSelector(labels)).toEqual('{foo="bar"}'); + }); + it('metric selector without labels', () => { + const labels: SelectableLabel[] = [{ name: '__name__', selected: true, values: [{ name: 'foo', selected: true }] }]; + expect(buildSelector(labels)).toEqual('foo{}'); + }); + it('selector with multiple metrics', () => { + const labels: SelectableLabel[] = [ + { + name: '__name__', + selected: true, + values: [ + { name: 'foo', selected: true }, + { name: 'bar', selected: true }, + ], + }, + ]; + expect(buildSelector(labels)).toEqual('{__name__=~"foo|bar"}'); + }); + it('metric selector with labels', () => { + const labels: SelectableLabel[] = [ + { name: '__name__', selected: true, values: [{ name: 'foo', selected: true }] }, + { name: 'bar', selected: true, values: [{ name: 'baz', selected: true }] }, + ]; + expect(buildSelector(labels)).toEqual('foo{bar="baz"}'); + }); +}); + +describe('facetLabels()', () => { + const possibleLabels = { + cluster: ['dev'], + namespace: ['alertmanager'], + }; + const labels: SelectableLabel[] = [ + { name: 'foo', selected: true, values: [{ name: 'bar' }] }, + { name: 'cluster', values: [{ name: 'dev' }, { name: 'ops' }, { name: 'prod' }] }, + { name: 'namespace', values: [{ name: 'alertmanager' }] }, + ]; + + it('returns no labels given an empty label set', () => { + expect(facetLabels([], {})).toEqual([]); + }); + + it('marks all labels as hidden when no labels are possible', () => { + const result = facetLabels(labels, {}); + expect(result.length).toEqual(labels.length); + expect(result[0].hidden).toBeTruthy(); + expect(result[0].values).toBeUndefined(); + }); + + it('keeps values as facetted when they are possible', () => { + const result = facetLabels(labels, possibleLabels); + expect(result.length).toEqual(labels.length); + expect(result[0].hidden).toBeTruthy(); + expect(result[0].values).toBeUndefined(); + expect(result[1].hidden).toBeFalsy(); + expect(result[1].values!.length).toBe(1); + expect(result[1].values![0].name).toBe('dev'); + }); + + it('does not facet out label values that are currently being facetted', () => { + const result = facetLabels(labels, possibleLabels, 'cluster'); + expect(result.length).toEqual(labels.length); + expect(result[0].hidden).toBeTruthy(); + expect(result[1].hidden).toBeFalsy(); + // 'cluster' is being facetted, should show all 3 options even though only 1 is possible + expect(result[1].values!.length).toBe(3); + expect(result[2].values!.length).toBe(1); + }); +}); + +describe('PrometheusMetricsBrowser', () => { + const setupProps = (): BrowserProps => { + const mockLanguageProvider = { + start: () => Promise.resolve(), + getLabelValues: (name: string) => { + switch (name) { + case 'label1': + return ['value1-1', 'value1-2']; + case 'label2': + return ['value2-1', 'value2-2']; + case 'label3': + return ['value3-1', 'value3-2']; + } + return []; + }, + fetchSeriesLabels: (selector: string) => { + switch (selector) { + case '{label1="value1-1"}': + return { label1: ['value1-1'], label2: ['value2-1'], label3: ['value3-1'] }; + case '{label1=~"value1-1|value1-2"}': + return { label1: ['value1-1', 'value1-2'], label2: ['value2-1'], label3: ['value3-1', 'value3-2'] }; + } + // Allow full set by default + return { + label1: ['value1-1', 'value1-2'], + label2: ['value2-1', 'value2-2'], + }; + }, + getLabelKeys: () => ['label1', 'label2', 'label3'], + }; + + const defaults: BrowserProps = { + theme: getTheme(), + onChange: () => {}, + autoSelect: 0, + languageProvider: (mockLanguageProvider as unknown) as PromQlLanguageProvider, + }; + + return defaults; + }; + + // Clear label selection manually because it's saved in localStorage + afterEach(() => { + const clearBtn = screen.getByLabelText('Selector clear button'); + userEvent.click(clearBtn); + }); + + it('renders and loader shows when empty, and then first set of labels', async () => { + const props = setupProps(); + render(); + // Loading appears and dissappears + screen.getByText(/Loading labels/); + await waitFor(() => { + expect(screen.queryByText(/Loading labels/)).not.toBeInTheDocument(); + }); + // Initial set of labels is available and not selected + expect(screen.queryByRole('option', { name: 'label1' })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'label1', selected: true })).not.toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'label2' })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'label2', selected: true })).not.toBeInTheDocument(); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{}'); + }); + + it('allows label and value selection/deselection', async () => { + const props = setupProps(); + render(); + // Selecting label2 + const label2 = await screen.findByRole('option', { name: /label2/, selected: false }); + expect(screen.queryByRole('list', { name: /Values/ })).not.toBeInTheDocument(); + userEvent.click(label2); + expect(screen.queryByRole('option', { name: /label2/, selected: true })).toBeInTheDocument(); + // List of values for label2 appears + expect(await screen.findAllByRole('list')).toHaveLength(1); + expect(screen.queryByLabelText(/Values for/)).toHaveTextContent('label2'); + expect(screen.queryByRole('option', { name: 'value2-1' })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'value2-2' })).toBeInTheDocument(); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{}'); + // Selecting label1, list for its values appears + const label1 = await screen.findByRole('option', { name: /label1/, selected: false }); + userEvent.click(label1); + expect(screen.queryByRole('option', { name: /label1/, selected: true })).toBeInTheDocument(); + await screen.findByLabelText('Values for label1'); + expect(await screen.findAllByRole('list', { name: /Values/ })).toHaveLength(2); + // Selecting value2-2 of label2 + const value = await screen.findByRole('option', { name: 'value2-2', selected: false }); + userEvent.click(value); + await screen.findByRole('option', { name: 'value2-2', selected: true }); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{label2="value2-2"}'); + // Selecting value2-1 of label2, both values now selected + const value2 = await screen.findByRole('option', { name: 'value2-1', selected: false }); + userEvent.click(value2); + // await screen.findByRole('option', {name: 'value2-1', selected: true}); + await screen.findByText('{label2=~"value2-1|value2-2"}'); + // Deselecting value2-2, one value should remain + const selectedValue = await screen.findByRole('option', { name: 'value2-2', selected: true }); + userEvent.click(selectedValue); + await screen.findByRole('option', { name: 'value2-1', selected: true }); + await screen.findByRole('option', { name: 'value2-2', selected: false }); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{label2="value2-1"}'); + // Selecting value from label1 for combined selector + const value1 = await screen.findByRole('option', { name: 'value1-2', selected: false }); + userEvent.click(value1); + await screen.findByRole('option', { name: 'value1-2', selected: true }); + await screen.findByText('{label1="value1-2",label2="value2-1"}'); + // Deselect label1 should remove label and value + const selectedLabel = (await screen.findAllByRole('option', { name: /label1/, selected: true }))[0]; + userEvent.click(selectedLabel); + await screen.findByRole('option', { name: /label1/, selected: false }); + expect(await screen.findAllByRole('list', { name: /Values/ })).toHaveLength(1); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{label2="value2-1"}'); + // Clear selector + const clearBtn = screen.getByLabelText('Selector clear button'); + userEvent.click(clearBtn); + await screen.findByRole('option', { name: /label2/, selected: false }); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{}'); + }); + + it('filters values by input text', async () => { + const props = setupProps(); + render(); + // Selecting label2 and label1 + const label2 = await screen.findByRole('option', { name: /label2/, selected: false }); + userEvent.click(label2); + const label1 = await screen.findByRole('option', { name: /label1/, selected: false }); + userEvent.click(label1); + await screen.findByLabelText('Values for label1'); + await screen.findByLabelText('Values for label2'); + expect(await screen.findAllByRole('option', { name: /value/ })).toHaveLength(4); + // Typing '1' to filter for values + userEvent.type(screen.getByLabelText('Filter expression for label values'), '1'); + expect(screen.getByLabelText('Filter expression for label values')).toHaveValue('1'); + expect(screen.queryByRole('option', { name: 'value2-2' })).not.toBeInTheDocument(); + expect(await screen.findAllByRole('option', { name: /value/ })).toHaveLength(3); + }); + + it('facets labels', async () => { + const props = setupProps(); + render(); + // Selecting label2 and label1 + const label2 = await screen.findByRole('option', { name: /label2/, selected: false }); + userEvent.click(label2); + const label1 = await screen.findByRole('option', { name: /label1/, selected: false }); + userEvent.click(label1); + await screen.findByLabelText('Values for label1'); + await screen.findByLabelText('Values for label2'); + expect(await screen.findAllByRole('option', { name: /value/ })).toHaveLength(4); + expect(screen.queryByRole('option', { name: /label3/ })).toHaveTextContent('label3'); + // Click value1-1 which triggers facetting for value3-x, and still show all value1-x + const value1 = await screen.findByRole('option', { name: 'value1-1', selected: false }); + userEvent.click(value1); + await waitForElementToBeRemoved(screen.queryByRole('option', { name: 'value2-2' })); + expect(screen.queryByRole('option', { name: 'value1-2' })).toBeInTheDocument(); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{label1="value1-1"}'); + expect(screen.queryByRole('option', { name: /label3/ })).toHaveTextContent('label3 (1)'); + // Click value1-2 for which facetting will allow more values for value3-x + const value12 = await screen.findByRole('option', { name: 'value1-2', selected: false }); + userEvent.click(value12); + await screen.findByRole('option', { name: 'value1-2', selected: true }); + userEvent.click(screen.getByRole('option', { name: /label3/ })); + await screen.findByLabelText('Values for label3'); + expect(screen.queryByRole('option', { name: 'value1-1', selected: true })).toBeInTheDocument(); + expect(screen.queryByRole('option', { name: 'value1-2', selected: true })).toBeInTheDocument(); + expect(screen.queryByLabelText('selector')).toHaveTextContent('{label1=~"value1-1|value1-2"}'); + expect(screen.queryAllByRole('option', { name: /label3/ })[0]).toHaveTextContent('label3 (2)'); + }); +}); diff --git a/public/app/plugins/datasource/prometheus/components/PrometheusMetricsBrowser.tsx b/public/app/plugins/datasource/prometheus/components/PrometheusMetricsBrowser.tsx new file mode 100644 index 00000000000..f6b83a569c9 --- /dev/null +++ b/public/app/plugins/datasource/prometheus/components/PrometheusMetricsBrowser.tsx @@ -0,0 +1,633 @@ +import React, { ChangeEvent } from 'react'; +import { Button, HorizontalGroup, Input, Label, LoadingPlaceholder, stylesFactory, withTheme } from '@grafana/ui'; +import PromQlLanguageProvider from '../language_provider'; +import { css, cx } from '@emotion/css'; +import store from 'app/core/store'; +import { FixedSizeList } from 'react-window'; + +import { GrafanaTheme } from '@grafana/data'; +import { Label as PromLabel } from './Label'; + +// Hard limit on labels to render +const MAX_LABEL_COUNT = 10000; +const MAX_VALUE_COUNT = 10000; +const EMPTY_SELECTOR = '{}'; +const METRIC_LABEL = '__name__'; +export const LAST_USED_LABELS_KEY = 'grafana.datasources.prometheus.browser.labels'; + +export interface BrowserProps { + languageProvider: PromQlLanguageProvider; + onChange: (selector: string) => void; + theme: GrafanaTheme; + autoSelect?: number; + hide?: () => void; +} + +interface BrowserState { + labels: SelectableLabel[]; + labelSearchTerm: string; + metricSearchTerm: string; + status: string; + error: string; + validationStatus: string; + valueSearchTerm: string; +} + +interface FacettableValue { + name: string; + selected?: boolean; +} + +export interface SelectableLabel { + name: string; + selected?: boolean; + loading?: boolean; + values?: FacettableValue[]; + hidden?: boolean; + facets?: number; +} + +export function buildSelector(labels: SelectableLabel[]): string { + let singleMetric = ''; + const selectedLabels = []; + for (const label of labels) { + if ((label.name === METRIC_LABEL || label.selected) && label.values && label.values.length > 0) { + const selectedValues = label.values.filter((value) => value.selected).map((value) => value.name); + if (selectedValues.length > 1) { + selectedLabels.push(`${label.name}=~"${selectedValues.join('|')}"`); + } else if (selectedValues.length === 1) { + if (label.name === METRIC_LABEL) { + singleMetric = selectedValues[0]; + } else { + selectedLabels.push(`${label.name}="${selectedValues[0]}"`); + } + } + } + } + return [singleMetric, '{', selectedLabels.join(','), '}'].join(''); +} + +export function facetLabels( + labels: SelectableLabel[], + possibleLabels: Record, + lastFacetted?: string +): SelectableLabel[] { + return labels.map((label) => { + const possibleValues = possibleLabels[label.name]; + if (possibleValues) { + let existingValues: FacettableValue[]; + if (label.name === lastFacetted && label.values) { + // Facetting this label, show all values + existingValues = label.values; + } else { + // Keep selection in other facets + const selectedValues: Set = new Set( + label.values?.filter((value) => value.selected).map((value) => value.name) || [] + ); + // Values for this label have not been requested yet, let's use the facetted ones as the initial values + existingValues = possibleValues.map((value) => ({ name: value, selected: selectedValues.has(value) })); + } + return { + ...label, + loading: false, + values: existingValues, + hidden: !possibleValues, + facets: existingValues.length, + }; + } + + // Label is facetted out, hide all values + return { ...label, loading: false, hidden: !possibleValues, values: undefined, facets: 0 }; + }); +} + +const getStyles = stylesFactory((theme: GrafanaTheme) => ({ + wrapper: css` + background-color: ${theme.colors.bg2}; + padding: ${theme.spacing.md}; + width: 100%; + `, + list: css` + margin-top: ${theme.spacing.sm}; + display: flex; + flex-wrap: wrap; + max-height: 200px; + overflow: auto; + `, + section: css` + & + & { + margin: ${theme.spacing.md} 0; + } + position: relative; + `, + selector: css` + font-family: ${theme.typography.fontFamily.monospace}; + margin-bottom: ${theme.spacing.sm}; + `, + status: css` + padding: ${theme.spacing.xs}; + color: ${theme.colors.textSemiWeak}; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + /* using absolute positioning because flex interferes with ellipsis */ + position: absolute; + width: 50%; + right: 0; + text-align: right; + transition: opacity 100ms linear; + opacity: 0; + `, + statusShowing: css` + opacity: 1; + `, + error: css` + color: ${theme.palette.brandDanger}; + `, + valueList: css` + margin-right: ${theme.spacing.sm}; + `, + valueListWrapper: css` + border-left: 1px solid ${theme.colors.border2}; + margin: ${theme.spacing.sm} 0; + padding: ${theme.spacing.sm} 0 ${theme.spacing.sm} ${theme.spacing.sm}; + `, + valueListArea: css` + display: flex; + flex-wrap: wrap; + margin-top: ${theme.spacing.sm}; + `, + valueTitle: css` + margin-left: -${theme.spacing.xs}; + margin-bottom: ${theme.spacing.sm}; + `, + validationStatus: css` + padding: ${theme.spacing.xs}; + margin-bottom: ${theme.spacing.sm}; + color: ${theme.colors.textStrong}; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + `, +})); + +/** + * TODO #33976: Remove duplicated code. The component is very similar to LokiLabelBrowser.tsx. Check if it's possible + * to create a single, generic component. + */ +export class UnthemedPrometheusMetricsBrowser extends React.Component { + state = { + labels: [] as SelectableLabel[], + labelSearchTerm: '', + metricSearchTerm: '', + status: 'Ready', + error: '', + validationStatus: '', + valueSearchTerm: '', + }; + + onChangeLabelSearch = (event: ChangeEvent) => { + this.setState({ labelSearchTerm: event.target.value }); + }; + + onChangeMetricSearch = (event: ChangeEvent) => { + this.setState({ metricSearchTerm: event.target.value }); + }; + + onChangeValueSearch = (event: ChangeEvent) => { + this.setState({ valueSearchTerm: event.target.value }); + }; + + onClickRunQuery = () => { + const selector = buildSelector(this.state.labels); + this.props.onChange(selector); + }; + + onClickRunRateQuery = () => { + const selector = buildSelector(this.state.labels); + const query = `rate(${selector}[$__interval])`; + this.props.onChange(query); + }; + + onClickClear = () => { + this.setState((state) => { + const labels: SelectableLabel[] = state.labels.map((label) => ({ + ...label, + values: undefined, + selected: false, + loading: false, + hidden: false, + facets: undefined, + })); + return { + labels, + labelSearchTerm: '', + metricSearchTerm: '', + status: '', + error: '', + validationStatus: '', + valueSearchTerm: '', + }; + }); + store.delete(LAST_USED_LABELS_KEY); + // Get metrics + this.fetchValues(METRIC_LABEL); + }; + + onClickLabel = (name: string, value: string | undefined, event: React.MouseEvent) => { + const label = this.state.labels.find((l) => l.name === name); + if (!label) { + return; + } + // Toggle selected state + const selected = !label.selected; + let nextValue: Partial = { selected }; + if (label.values && !selected) { + // Deselect all values if label was deselected + const values = label.values.map((value) => ({ ...value, selected: false })); + nextValue = { ...nextValue, facets: 0, values }; + } + // Resetting search to prevent empty results + this.setState({ labelSearchTerm: '' }); + this.updateLabelState(name, nextValue, '', () => this.doFacettingForLabel(name)); + }; + + onClickValue = (name: string, value: string | undefined, event: React.MouseEvent) => { + const label = this.state.labels.find((l) => l.name === name); + if (!label || !label.values) { + return; + } + // Resetting search to prevent empty results + this.setState({ labelSearchTerm: '' }); + // Toggling value for selected label, leaving other values intact + const values = label.values.map((v) => ({ ...v, selected: v.name === value ? !v.selected : v.selected })); + this.updateLabelState(name, { values }, '', () => this.doFacetting(name)); + }; + + onClickMetric = (name: string, value: string | undefined, event: React.MouseEvent) => { + // Finding special metric label + const label = this.state.labels.find((l) => l.name === name); + if (!label || !label.values) { + return; + } + // Resetting search to prevent empty results + this.setState({ metricSearchTerm: '' }); + // Toggling value for selected label, leaving other values intact + const values = label.values.map((v) => ({ + ...v, + selected: v.name === value || v.selected ? !v.selected : v.selected, + })); + // Toggle selected state of special metrics label + const selected = values.some((v) => v.selected); + this.updateLabelState(name, { selected, values }, '', () => this.doFacetting(name)); + }; + + onClickValidate = () => { + const selector = buildSelector(this.state.labels); + this.validateSelector(selector); + }; + + updateLabelState(name: string, updatedFields: Partial, status = '', cb?: () => void) { + this.setState((state) => { + const labels: SelectableLabel[] = state.labels.map((label) => { + if (label.name === name) { + return { ...label, ...updatedFields }; + } + return label; + }); + // New status overrides errors + const error = status ? '' : state.error; + return { labels, status, error, validationStatus: '' }; + }, cb); + } + + componentDidMount() { + const { languageProvider } = this.props; + if (languageProvider) { + const selectedLabels: string[] = store.getObject(LAST_USED_LABELS_KEY, []); + languageProvider.start().then(() => { + let rawLabels: string[] = languageProvider.getLabelKeys(); + // TODO too-many-metrics + if (rawLabels.length > MAX_LABEL_COUNT) { + const error = `Too many labels found (showing only ${MAX_LABEL_COUNT} of ${rawLabels.length})`; + rawLabels = rawLabels.slice(0, MAX_LABEL_COUNT); + this.setState({ error }); + } + // Get metrics + this.fetchValues(METRIC_LABEL); + // Auto-select previously selected labels + const labels: SelectableLabel[] = rawLabels.map((label, i, arr) => ({ + name: label, + selected: selectedLabels.includes(label), + loading: false, + })); + // Pre-fetch values for selected labels + this.setState({ labels }, () => { + this.state.labels.forEach((label) => { + if (label.selected) { + this.fetchValues(label.name); + } + }); + }); + }); + } + } + + doFacettingForLabel(name: string) { + const label = this.state.labels.find((l) => l.name === name); + if (!label) { + return; + } + const selectedLabels = this.state.labels.filter((label) => label.selected).map((label) => label.name); + store.setObject(LAST_USED_LABELS_KEY, selectedLabels); + if (label.selected) { + // Refetch values for newly selected label... + if (!label.values) { + this.fetchValues(name); + } + } else { + // Only need to facet when deselecting labels + this.doFacetting(); + } + } + + doFacetting = (lastFacetted?: string) => { + const selector = buildSelector(this.state.labels); + if (selector === EMPTY_SELECTOR) { + // Clear up facetting + const labels: SelectableLabel[] = this.state.labels.map((label) => { + return { ...label, facets: 0, values: undefined, hidden: false }; + }); + this.setState({ labels }, () => { + // Get fresh set of values + this.state.labels.forEach( + (label) => (label.selected || label.name === METRIC_LABEL) && this.fetchValues(label.name) + ); + }); + } else { + // Do facetting + this.fetchSeries(selector, lastFacetted); + } + }; + + async fetchValues(name: string) { + const { languageProvider } = this.props; + this.updateLabelState(name, { loading: true }, `Fetching values for ${name}`); + try { + let rawValues = await languageProvider.getLabelValues(name); + if (rawValues.length > MAX_VALUE_COUNT) { + const error = `Too many values for ${name} (showing only ${MAX_VALUE_COUNT} of ${rawValues.length})`; + rawValues = rawValues.slice(0, MAX_VALUE_COUNT); + this.setState({ error }); + } + const values: FacettableValue[] = rawValues.map((value) => ({ name: value })); + this.updateLabelState(name, { values, loading: false }, ''); + } catch (error) { + console.error(error); + } + } + + async fetchSeries(selector: string, lastFacetted?: string) { + const { languageProvider } = this.props; + if (lastFacetted) { + this.updateLabelState(lastFacetted, { loading: true }, `Facetting labels for ${selector}`); + } + try { + const possibleLabels = await languageProvider.fetchSeriesLabels(selector, true); + if (Object.keys(possibleLabels).length === 0) { + // Sometimes the backend does not return a valid set + console.error('No results for label combination, but should not occur.'); + this.setState({ error: `Facetting failed for ${selector}` }); + return; + } + const labels: SelectableLabel[] = facetLabels(this.state.labels, possibleLabels, lastFacetted); + this.setState({ labels, error: '' }); + if (lastFacetted) { + this.updateLabelState(lastFacetted, { loading: false }); + } + } catch (error) { + console.error(error); + } + } + + async validateSelector(selector: string) { + const { languageProvider } = this.props; + this.setState({ validationStatus: `Validating selector ${selector}`, error: '' }); + const streams = await languageProvider.fetchSeries(selector); + this.setState({ validationStatus: `Selector is valid (${streams.length} streams found)` }); + } + + render() { + const { theme } = this.props; + const { labels, labelSearchTerm, metricSearchTerm, status, error, validationStatus, valueSearchTerm } = this.state; + const styles = getStyles(theme); + if (labels.length === 0) { + return ( +
+ +
+ ); + } + + // Filter metrics + let metrics = labels.find((label) => label.name === METRIC_LABEL); + if (metrics && metricSearchTerm) { + // TODO extract from render() and debounce + metrics = { + ...metrics, + values: metrics.values?.filter((value) => value.selected || value.name.includes(metricSearchTerm)), + }; + } + + // Filter labels + let nonMetricLabels = labels.filter((label) => !label.hidden && label.name !== METRIC_LABEL); + if (labelSearchTerm) { + // TODO extract from render() and debounce + nonMetricLabels = nonMetricLabels.filter((label) => label.selected || label.name.includes(labelSearchTerm)); + } + + // Filter non-metric label values + let selectedLabels = nonMetricLabels.filter((label) => label.selected && label.values); + if (valueSearchTerm) { + // TODO extract from render() and debounce + selectedLabels = selectedLabels.map((label) => ({ + ...label, + values: label.values?.filter((value) => value.selected || value.name.includes(valueSearchTerm)), + })); + } + const selector = buildSelector(this.state.labels); + const empty = selector === EMPTY_SELECTOR; + return ( +
+ +
+
+ +
+ +
+
+ (metrics!.values as FacettableValue[])[i].name} + width={300} + className={styles.valueList} + > + {({ index, style }) => { + const value = metrics?.values?.[index]; + if (!value) { + return null; + } + return ( +
+ +
+ ); + }} + +
+
+
+ +
+
+ +
+ +
+
+ {nonMetricLabels.map((label) => ( +
+
+
+ +
+ +
+
+ {selectedLabels.map((label) => ( +
+
+
+ (label.values as FacettableValue[])[i].name} + width={200} + className={styles.valueList} + > + {({ index, style }) => { + const value = label.values?.[index]; + if (!value) { + return null; + } + return ( +
+ +
+ ); + }} + +
+ ))} +
+
+
+ + +
+ +
+ {selector} +
+ {validationStatus &&
{validationStatus}
} + + + + + +
+ {error || status} +
+ +
+
+ ); + } +} + +export const PrometheusMetricsBrowser = withTheme(UnthemedPrometheusMetricsBrowser); diff --git a/public/app/plugins/datasource/prometheus/components/__snapshots__/PromExploreQueryEditor.test.tsx.snap b/public/app/plugins/datasource/prometheus/components/__snapshots__/PromExploreQueryEditor.test.tsx.snap index 24118aab039..9f69a2cb974 100644 --- a/public/app/plugins/datasource/prometheus/components/__snapshots__/PromExploreQueryEditor.test.tsx.snap +++ b/public/app/plugins/datasource/prometheus/components/__snapshots__/PromExploreQueryEditor.test.tsx.snap @@ -7,6 +7,8 @@ exports[`PromExploreQueryEditor should render component 1`] = ` datasource={ Object { "languageProvider": Object { + "getLabelKeys": [Function], + "metrics": Array [], "syntax": [Function], }, } @@ -63,6 +65,8 @@ exports[`PromExploreQueryEditor should render component 1`] = ` datasource={ Object { "languageProvider": Object { + "getLabelKeys": [Function], + "metrics": Array [], "syntax": [Function], }, } diff --git a/public/app/plugins/datasource/prometheus/datasource.ts b/public/app/plugins/datasource/prometheus/datasource.ts index 9a95731b414..1d5f15f398e 100644 --- a/public/app/plugins/datasource/prometheus/datasource.ts +++ b/public/app/plugins/datasource/prometheus/datasource.ts @@ -804,11 +804,11 @@ export class PrometheusDatasource extends DataSourceApi return Math.ceil(date.valueOf() / 1000); } - getTimeRange(): { start: number; end: number } { + getTimeRangeParams(): { start: string; end: string } { const range = this.timeSrv.timeRange(); return { - start: this.getPrometheusTime(range.from, false), - end: this.getPrometheusTime(range.to, true), + start: this.getPrometheusTime(range.from, false).toString(), + end: this.getPrometheusTime(range.to, true).toString(), }; } diff --git a/public/app/plugins/datasource/prometheus/language_provider.test.ts b/public/app/plugins/datasource/prometheus/language_provider.test.ts index 2bbc9ddf89c..e29ca883f39 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.test.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.test.ts @@ -10,7 +10,7 @@ import { SearchFunctionType } from '@grafana/ui'; describe('Language completion provider', () => { const datasource: PrometheusDatasource = ({ metadataRequest: () => ({ data: { data: [] as any[] } }), - getTimeRange: () => ({ start: 0, end: 1 }), + getTimeRangeParams: () => ({ start: '0', end: '1' }), } as any) as PrometheusDatasource; describe('cleanText', () => { @@ -249,7 +249,7 @@ describe('Language completion provider', () => { it('returns label suggestions on label context and metric', async () => { const datasources: PrometheusDatasource = ({ metadataRequest: () => ({ data: { data: [{ __name__: 'metric', bar: 'bazinga' }] as any[] } }), - getTimeRange: () => ({ start: 0, end: 1 }), + getTimeRangeParams: () => ({ start: '0', end: '1' }), } as any) as PrometheusDatasource; const instance = new LanguageProvider(datasources); const value = Plain.deserialize('metric{}'); @@ -282,7 +282,7 @@ describe('Language completion provider', () => { ], }, }), - getTimeRange: () => ({ start: 0, end: 1 }), + getTimeRangeParams: () => ({ start: '0', end: '1' }), } as any) as PrometheusDatasource; const instance = new LanguageProvider(datasource); const value = Plain.deserialize('{job1="foo",job2!="foo",job3=~"foo",__name__="metric",}'); @@ -519,7 +519,7 @@ describe('Language completion provider', () => { it('does not re-fetch default labels', async () => { const datasource: PrometheusDatasource = ({ metadataRequest: jest.fn(() => ({ data: { data: [] as any[] } })), - getTimeRange: jest.fn(() => ({ start: 0, end: 1 })), + getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })), } as any) as PrometheusDatasource; const instance = new LanguageProvider(datasource); @@ -545,7 +545,7 @@ describe('Language completion provider', () => { it('does not issue any metadata requests when lookup is disabled', async () => { const datasource: PrometheusDatasource = ({ metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })), - getTimeRange: jest.fn(() => ({ start: 0, end: 1 })), + getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })), lookupsDisabled: true, } as any) as PrometheusDatasource; const instance = new LanguageProvider(datasource); @@ -568,7 +568,7 @@ describe('Language completion provider', () => { it('issues metadata requests when lookup is not disabled', async () => { const datasource: PrometheusDatasource = ({ metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })), - getTimeRange: jest.fn(() => ({ start: 0, end: 1 })), + getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })), lookupsDisabled: false, } as any) as PrometheusDatasource; const instance = new LanguageProvider(datasource); diff --git a/public/app/plugins/datasource/prometheus/language_provider.ts b/public/app/plugins/datasource/prometheus/language_provider.ts index c8ae4f535c2..aa88e221acb 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.ts @@ -69,6 +69,8 @@ export default class PromQlLanguageProvider extends LanguageProvider { metricsMetadata?: PromMetricsMetadata; startTask: Promise; datasource: PrometheusDatasource; + labelKeys: string[]; + labelFetchTs: number; /** * Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does @@ -115,20 +117,19 @@ export default class PromQlLanguageProvider extends LanguageProvider { return []; } - const tRange = this.datasource.getTimeRange(); - const params = { - start: tRange['start'].toString(), - end: tRange['end'].toString(), - }; - const url = `/api/v1/label/__name__/values`; - - this.metrics = await this.request(url, [], params); + // TODO #33976: make those requests parallel + await this.fetchLabels(); + this.metrics = await this.fetchLabelValues('__name__'); this.metricsMetadata = fixSummariesMetadata(await this.request('/api/v1/metadata', {})); this.processHistogramMetrics(this.metrics); return []; }; + getLabelKeys(): string[] { + return this.labelKeys; + } + processHistogramMetrics = (data: string[]) => { const { values } = processHistogramLabels(data); @@ -308,12 +309,13 @@ export default class PromQlLanguageProvider extends LanguageProvider { const selector = parseSelector(selectorString, selectorString.length - 2).selector; - const labelValues = await this.getLabelValues(selector); - if (labelValues) { - const limitInfo = addLimitInfo(labelValues[0]); + const series = await this.getSeries(selector); + const labelKeys = Object.keys(series); + if (labelKeys.length > 0) { + const limitInfo = addLimitInfo(labelKeys); suggestions.push({ label: `Labels${limitInfo}`, - items: Object.keys(labelValues).map(wrapLabel), + items: labelKeys.map(wrapLabel), searchFunctionType: SearchFunctionType.Fuzzy, }); } @@ -360,13 +362,13 @@ export default class PromQlLanguageProvider extends LanguageProvider { const containsMetric = selector.includes('__name__='); const existingKeys = parsedSelector ? parsedSelector.labelKeys : []; - let labelValues; + let series: Record = {}; // Query labels for selector if (selector) { - labelValues = await this.getLabelValues(selector, !containsMetric); + series = await this.getSeries(selector, !containsMetric); } - if (!labelValues) { + if (Object.keys(series).length === 0) { console.warn(`Server did not return any values for selector = ${selector}`); return { suggestions }; } @@ -375,18 +377,18 @@ export default class PromQlLanguageProvider extends LanguageProvider { if ((text && isValueStart) || wrapperClasses.includes('attr-value')) { // Label values - if (labelKey && labelValues[labelKey]) { + if (labelKey && series[labelKey]) { context = 'context-label-values'; - const limitInfo = addLimitInfo(labelValues[labelKey]); + const limitInfo = addLimitInfo(series[labelKey]); suggestions.push({ label: `Label values for "${labelKey}"${limitInfo}`, - items: labelValues[labelKey].map(wrapLabel), + items: series[labelKey].map(wrapLabel), searchFunctionType: SearchFunctionType.Fuzzy, }); } } else { // Label keys - const labelKeys = labelValues ? Object.keys(labelValues) : containsMetric ? null : DEFAULT_KEYS; + const labelKeys = series ? Object.keys(series) : containsMetric ? null : DEFAULT_KEYS; if (labelKeys) { const possibleKeys = difference(labelKeys, existingKeys); @@ -407,34 +409,49 @@ export default class PromQlLanguageProvider extends LanguageProvider { return { context, suggestions }; }; - async getLabelValues(selector: string, withName?: boolean) { + async getSeries(selector: string, withName?: boolean): Promise> { if (this.datasource.lookupsDisabled) { - return undefined; + return {}; } try { if (selector === EMPTY_SELECTOR) { - return await this.fetchDefaultLabels(); + return await this.fetchDefaultSeries(); } else { return await this.fetchSeriesLabels(selector, withName); } } catch (error) { // TODO: better error handling console.error(error); - return undefined; + return {}; } } - fetchLabelValues = async (key: string): Promise> => { - const tRange = this.datasource.getTimeRange(); - const params = { - start: tRange['start'].toString(), - end: tRange['end'].toString(), - }; + fetchLabelValues = async (key: string): Promise => { + const params = this.datasource.getTimeRangeParams(); const url = `/api/v1/label/${key}/values`; - const data = await this.request(url, [], params); - return { [key]: data }; + return await this.request(url, [], params); }; + async getLabelValues(key: string): Promise { + return await this.fetchLabelValues(key); + } + + /** + * Fetches all label keys + */ + async fetchLabels(): Promise { + const url = '/api/v1/labels'; + const params = this.datasource.getTimeRangeParams(); + this.labelFetchTs = Date.now().valueOf(); + + const res = await this.request(url, [], params); + if (Array.isArray(res)) { + this.labelKeys = res.slice().sort(); + } + + return []; + } + /** * Fetch labels for a series. This is cached by it's args but also by the global timeRange currently selected as * they can change over requested time. @@ -442,11 +459,10 @@ export default class PromQlLanguageProvider extends LanguageProvider { * @param withName */ fetchSeriesLabels = async (name: string, withName?: boolean): Promise> => { - const tRange = this.datasource.getTimeRange(); + const range = this.datasource.getTimeRangeParams(); const urlParams = { + ...range, 'match[]': name, - start: tRange['start'].toString(), - end: tRange['end'].toString(), }; const url = `/api/v1/series`; // Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals. @@ -455,8 +471,8 @@ export default class PromQlLanguageProvider extends LanguageProvider { // when user does not the newest values for a minute if already cached. const cacheParams = new URLSearchParams({ 'match[]': name, - start: roundSecToMin(tRange['start']).toString(), - end: roundSecToMin(tRange['end']).toString(), + start: roundSecToMin(parseInt(range.start, 10)).toString(), + end: roundSecToMin(parseInt(range.end, 10)).toString(), withName: withName ? 'true' : 'false', }); @@ -471,13 +487,24 @@ export default class PromQlLanguageProvider extends LanguageProvider { return value; }; + /** + * Fetch series for a selector. Use this for raw results. Use fetchSeriesLabels() to get labels. + * @param match + */ + fetchSeries = async (match: string): Promise>> => { + const url = '/api/v1/series'; + const range = this.datasource.getTimeRangeParams(); + const params = { ...range, match }; + return await this.request(url, {}, params); + }; + /** * Fetch this only one as we assume this won't change over time. This is cached differently from fetchSeriesLabels * because we can cache more aggressively here and also we do not want to invalidate this cache the same way as in * fetchSeriesLabels. */ - fetchDefaultLabels = once(async () => { + fetchDefaultSeries = once(async () => { const values = await Promise.all(DEFAULT_KEYS.map((key) => this.fetchLabelValues(key))); - return values.reduce((acc, value) => ({ ...acc, ...value }), {}); + return DEFAULT_KEYS.reduce((acc, key, i) => ({ ...acc, [key]: values[i] }), {}); }); } diff --git a/public/app/plugins/panel/timeseries/TimeSeriesPanel.tsx b/public/app/plugins/panel/timeseries/TimeSeriesPanel.tsx index c59b50c546f..c66a37bbbc4 100644 --- a/public/app/plugins/panel/timeseries/TimeSeriesPanel.tsx +++ b/public/app/plugins/panel/timeseries/TimeSeriesPanel.tsx @@ -1,4 +1,4 @@ -import { DashboardCursorSync, Field, PanelProps } from '@grafana/data'; +import { anySeriesWithTimeField, DashboardCursorSync, Field, PanelProps } from '@grafana/data'; import { TooltipDisplayMode, usePanelContext, TimeSeries, TooltipPlugin, ZoomPlugin } from '@grafana/ui'; import { getFieldLinksForExplore } from 'app/features/explore/utils/links'; import React from 'react'; @@ -19,12 +19,12 @@ export const TimeSeriesPanel: React.FC = ({ onChangeTimeRange, replaceVariables, }) => { + const { sync } = usePanelContext(); + const getFieldLinks = (field: Field, rowIndex: number) => { return getFieldLinksForExplore({ field, rowIndex, range: timeRange }); }; - const { sync } = usePanelContext(); - if (!data || !data.series?.length) { return (
@@ -33,6 +33,14 @@ export const TimeSeriesPanel: React.FC = ({ ); } + if (!anySeriesWithTimeField(data.series)) { + return ( +
+

Missing time field in the data

+
+ ); + } + return ( : {{build.link}}\nAuthor: {{build.author}}', }, @@ -137,9 +139,7 @@ def init_steps(edition, platform, ver_mode, is_downstream=False, install_deps=Tr 'name': 'clone', 'image': build_image, 'environment': { - 'GITHUB_TOKEN': { - 'from_secret': 'github_token', - }, + 'GITHUB_TOKEN': from_secret(github_token), }, 'commands': download_grabpl_cmds + [ 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"', @@ -192,9 +192,7 @@ def enterprise_downstream_step(edition): 'image': 'grafana/drone-downstream', 'settings': { 'server': 'https://drone.grafana.net', - 'token': { - 'from_secret': 'drone_token', - }, + 'token': from_secret('drone_token'), 'repositories': [ 'grafana/grafana-enterprise@main', ], @@ -300,9 +298,7 @@ def publish_storybook_step(edition, ver_mode): 'end-to-end-tests', ], 'environment': { - 'GCP_KEY': { - 'from_secret': 'gcp_key', - }, + 'GCP_KEY': from_secret('gcp_key'), }, 'commands': commands, } @@ -315,9 +311,7 @@ def upload_cdn(edition): 'package' + enterprise2_sfx(edition), ], 'environment': { - 'GCP_GRAFANA_UPLOAD_KEY': { - 'from_secret': 'gcp_key', - }, + 'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'), }, 'commands': [ './bin/grabpl upload-cdn --edition {} --bucket "grafana-static-assets"'.format(edition), @@ -332,9 +326,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False): # TODO: Convert number of jobs to percentage if ver_mode == 'release': env = { - 'GITHUB_TOKEN': { - 'from_secret': 'github_token', - }, + 'GITHUB_TOKEN': from_secret(github_token), } cmds = [ './bin/grabpl build-backend --jobs 8 --edition {} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise ${{DRONE_TAG}}'.format( @@ -343,9 +335,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False): ] elif ver_mode == 'test-release': env = { - 'GITHUB_TOKEN': { - 'from_secret': 'github_token', - }, + 'GITHUB_TOKEN': from_secret(github_token), } cmds = [ './bin/grabpl build-backend --jobs 8 --edition {} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise {}'.format( @@ -424,9 +414,7 @@ def build_frontend_docs_step(edition): def build_plugins_step(edition, sign=False): if sign: env = { - 'GRAFANA_API_KEY': { - 'from_secret': 'grafana_api_key', - }, + 'GRAFANA_API_KEY': from_secret('grafana_api_key'), } sign_args = ' --sign --signing-admin' else: @@ -489,9 +477,7 @@ def frontend_metrics_step(edition): 'initialize', ], 'environment': { - 'GRAFANA_MISC_STATS_API_KEY': { - 'from_secret': 'grafana_misc_stats_api_key', - }, + 'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'), }, 'failure': 'ignore', 'commands': [ @@ -572,21 +558,11 @@ def package_step(edition, ver_mode, variants=None, is_downstream=False): if ver_mode in ('main', 'release', 'test-release', 'release-branch'): sign_args = ' --sign' env = { - 'GRAFANA_API_KEY': { - 'from_secret': 'grafana_api_key', - }, - 'GITHUB_TOKEN': { - 'from_secret': 'github_token', - }, - 'GPG_PRIV_KEY': { - 'from_secret': 'gpg_priv_key', - }, - 'GPG_PUB_KEY': { - 'from_secret': 'gpg_pub_key', - }, - 'GPG_KEY_PASSWORD': { - 'from_secret': 'gpg_key_password', - }, + 'GRAFANA_API_KEY': from_secret('grafana_api_key'), + 'GITHUB_TOKEN': from_secret(github_token), + 'GPG_PRIV_KEY': from_secret('gpg_priv_key'), + 'GPG_PUB_KEY': from_secret('gpg_pub_key'), + 'GPG_KEY_PASSWORD': from_secret('gpg_key_password'), } test_args = '' else: @@ -724,12 +700,8 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis } if publish: - settings['username'] = { - 'from_secret': 'docker_user', - } - settings['password'] = { - 'from_secret': 'docker_password', - } + settings['username'] = from_secret('docker_user') + settings['password'] = from_secret('docker_password') if archs: settings['archs'] = ','.join(archs) return { @@ -832,33 +804,31 @@ def release_canary_npm_packages_step(edition): 'end-to-end-tests', ], 'environment': { - 'GITHUB_PACKAGE_TOKEN': { - 'from_secret': 'github_package_token', - }, + 'GITHUB_PACKAGE_TOKEN': from_secret('github_package_token'), }, 'commands': [ './scripts/circle-release-canary-packages.sh', ], } -def deploy_to_kubernetes_step(edition, is_downstream=False): +def push_to_deployment_tools_step(edition, is_downstream=False): if edition != 'enterprise' or not is_downstream: return None return { - 'name': 'deploy-to-kubernetes', - 'image': alpine_image, + 'name': 'push-to-deployment_tools', + 'image': deploy_docker_image, 'depends_on': [ 'build-docker-images', + # This step should have all the dependencies required for packaging, and should generate + # dist/grafana.version + 'gen-version', ], - 'environment': { - 'CIRCLE_TOKEN': { - 'from_secret': 'deployment_tools_circle_token', - }, + 'settings': { + 'github_token': from_secret(github_token), + 'images_file': './deployment_tools_config.json', + 'docker_tag_file': './dist/grafana.version' }, - 'commands': [ - './bin/grabpl deploy-to-k8s', - ], } def enterprise2_sfx(edition): @@ -894,9 +864,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False): 'image': publish_image, 'depends_on': dependencies, 'environment': { - 'GCP_GRAFANA_UPLOAD_KEY': { - 'from_secret': 'gcp_key', - }, + 'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'), }, 'commands': [cmd,], } @@ -930,21 +898,11 @@ def publish_packages_step(edition, ver_mode, is_downstream=False): 'initialize', ], 'environment': { - 'GRAFANA_COM_API_KEY': { - 'from_secret': 'grafana_api_key', - }, - 'GCP_KEY': { - 'from_secret': 'gcp_key', - }, - 'GPG_PRIV_KEY': { - 'from_secret': 'gpg_priv_key', - }, - 'GPG_PUB_KEY': { - 'from_secret': 'gpg_pub_key', - }, - 'GPG_KEY_PASSWORD': { - 'from_secret': 'gpg_key_password', - }, + 'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'), + 'GCP_KEY': from_secret('gcp_key'), + 'GPG_PRIV_KEY': from_secret('gpg_priv_key'), + 'GPG_PUB_KEY': from_secret('gpg_pub_key'), + 'GPG_KEY_PASSWORD': from_secret('gpg_key_password'), }, 'commands': [ 'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json', @@ -1017,9 +975,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): 'name': 'build-windows-installer', 'image': wix_image, 'environment': { - 'GCP_KEY': { - 'from_secret': 'gcp_key', - }, + 'GCP_KEY': from_secret('gcp_key'), }, 'commands': installer_commands, 'depends_on': [ @@ -1053,9 +1009,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False): 'name': 'clone', 'image': wix_image, 'environment': { - 'GITHUB_TOKEN': { - 'from_secret': 'github_token', - }, + 'GITHUB_TOKEN': from_secret(github_token), }, 'commands': download_grabpl_cmds + clone_cmds, }) diff --git a/scripts/main.star b/scripts/main.star index 744fbbbbfea..7041c29335b 100644 --- a/scripts/main.star +++ b/scripts/main.star @@ -29,7 +29,7 @@ load( 'publish_storybook_step', 'release_canary_npm_packages_step', 'upload_packages_step', - 'deploy_to_kubernetes_step', + 'push_to_deployment_tools_step', 'publish_packages_step', 'notify_pipeline', 'integration_test_services', @@ -87,7 +87,7 @@ def get_steps(edition, is_downstream=False): steps.extend([ release_canary_npm_packages_step(edition), upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), - deploy_to_kubernetes_step(edition=edition, is_downstream=is_downstream), + push_to_deployment_tools_step(edition=edition, is_downstream=is_downstream), upload_cdn(edition=edition) ]) diff --git a/scripts/vault.star b/scripts/vault.star new file mode 100644 index 00000000000..d49f4d2dfbd --- /dev/null +++ b/scripts/vault.star @@ -0,0 +1,23 @@ +pull_secret = 'dockerconfigjson' +github_token = 'github_token' + +def from_secret(secret): + return { + 'from_secret': secret + } + +def vault_secret(name, path, key): + return { + 'kind': 'secret', + 'name': name, + 'get': { + 'path': path, + 'name': key, + } + } + +def secrets(): + return [ + vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'), + vault_secret(github_token, 'infra/data/ci/github/grafanabot', 'pat') + ]