From f20053e4b6cfda48020911ba74cac2a3c8e50ec4 Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Wed, 31 Jan 2024 03:30:27 -0800 Subject: [PATCH 01/84] DashboardScene: Allow editing panel links (#81560) * Panel links supplier for VizPanel * Update panel links behavior * Allow editing panel links * Update so that single link is rendered without a dropdown * Serialise links in scene -> save model transformation * Betterer fix * Fix inspect json tab test --- .betterer.results | 3 +- .../inspect/InspectJsonTab.test.tsx | 2 + .../dashboard-scene/scene/PanelLinks.tsx | 30 +++++++++-- .../scene/PanelMenuBehavior.tsx | 53 +++++++++++-------- .../transformSceneToSaveModel.test.ts.snap | 8 +++ .../transformSaveModelToScene.ts | 16 +++--- .../transformSceneToSaveModel.test.ts | 33 ++++++++++++ .../transformSceneToSaveModel.ts | 5 ++ .../utils/dashboardSceneGraph.test.ts | 25 +++++++++ .../utils/dashboardSceneGraph.ts | 16 +++++- .../dashboard-scene/utils/test-utils.ts | 7 ++- .../PanelEditor/getPanelFrameOptions.tsx | 50 +++++++++-------- .../panel/panellinks/linkSuppliers.ts | 21 ++++++++ 13 files changed, 208 insertions(+), 61 deletions(-) diff --git a/.betterer.results b/.betterer.results index fcaefeed57e..ba53b876c21 100644 --- a/.betterer.results +++ b/.betterer.results @@ -2442,7 +2442,8 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "2"], [0, 0, 0, "Do not use any type assertions.", "3"], [0, 0, 0, "Do not use any type assertions.", "4"], - [0, 0, 0, "Do not use any type assertions.", "5"] + [0, 0, 0, "Do not use any type assertions.", "5"], + [0, 0, 0, "Do not use any type assertions.", "6"] ], "public/app/features/dashboard-scene/settings/variables/components/VariableSelectField.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] diff --git a/public/app/features/dashboard-scene/inspect/InspectJsonTab.test.tsx b/public/app/features/dashboard-scene/inspect/InspectJsonTab.test.tsx index b1b91bc6ec1..903ce6daddc 100644 --- a/public/app/features/dashboard-scene/inspect/InspectJsonTab.test.tsx +++ b/public/app/features/dashboard-scene/inspect/InspectJsonTab.test.tsx @@ -12,6 +12,7 @@ import { import { getStandardTransformers } from 'app/features/transformers/standardTransformers'; import { DashboardScene } from '../scene/DashboardScene'; +import { VizPanelLinks, VizPanelLinksMenu } from '../scene/PanelLinks'; import { activateFullSceneTree } from '../utils/test-utils'; import { findVizPanelByKey } from '../utils/utils'; @@ -90,6 +91,7 @@ async function buildTestScene() { title: 'Panel A', pluginId: 'table', key: 'panel-12', + titleItems: [new VizPanelLinks({ menu: new VizPanelLinksMenu({}) })], $data: new SceneDataTransformer({ transformations: [ { diff --git a/public/app/features/dashboard-scene/scene/PanelLinks.tsx b/public/app/features/dashboard-scene/scene/PanelLinks.tsx index 2a0bcd7501e..62643f15e38 100644 --- a/public/app/features/dashboard-scene/scene/PanelLinks.tsx +++ b/public/app/features/dashboard-scene/scene/PanelLinks.tsx @@ -1,10 +1,13 @@ import React from 'react'; -import { LinkModel } from '@grafana/data'; -import { SceneComponentProps, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; -import { Dropdown, Menu, ToolbarButton } from '@grafana/ui'; +import { DataLink, LinkModel } from '@grafana/data'; +import { SceneComponentProps, SceneObjectBase, SceneObjectState, VizPanel } from '@grafana/scenes'; +import { Dropdown, Icon, Menu, PanelChrome, ToolbarButton } from '@grafana/ui'; + +import { getPanelLinks } from './PanelMenuBehavior'; interface VizPanelLinksState extends SceneObjectState { + rawLinks?: DataLink[]; links?: LinkModel[]; menu: VizPanelLinksMenu; } @@ -14,7 +17,24 @@ export class VizPanelLinks extends SceneObjectBase { } function VizPanelLinksRenderer({ model }: SceneComponentProps) { - const { menu } = model.useState(); + const { menu, rawLinks } = model.useState(); + + if (!(model.parent instanceof VizPanel)) { + throw new Error('VizPanelLinks must be a child of VizPanel'); + } + + if (!rawLinks || rawLinks.length === 0) { + return null; + } + + if (rawLinks.length === 1) { + const link = getPanelLinks(model.parent)[0]; + return ( + + + + ); + } return ( ) { ); } -export class VizPanelLinksMenu extends SceneObjectBase> { +export class VizPanelLinksMenu extends SceneObjectBase> { static Component = VizPanelLinksMenuRenderer; } diff --git a/public/app/features/dashboard-scene/scene/PanelMenuBehavior.tsx b/public/app/features/dashboard-scene/scene/PanelMenuBehavior.tsx index d85d6dc7564..757c6599a64 100644 --- a/public/app/features/dashboard-scene/scene/PanelMenuBehavior.tsx +++ b/public/app/features/dashboard-scene/scene/PanelMenuBehavior.tsx @@ -10,9 +10,8 @@ import { config, getPluginLinkExtensions, locationService } from '@grafana/runti import { LocalValueVariable, SceneGridRow, VizPanel, VizPanelMenu, sceneGraph } from '@grafana/scenes'; import { DataQuery } from '@grafana/schema'; import { t } from 'app/core/internationalization'; -import { PanelModel } from 'app/features/dashboard/state'; import { InspectTab } from 'app/features/inspector/types'; -import { getPanelLinksSupplier } from 'app/features/panel/panellinks/linkSuppliers'; +import { getScenePanelLinksSupplier } from 'app/features/panel/panellinks/linkSuppliers'; import { createExtensionSubMenu } from 'app/features/plugins/extensions/utils'; import { addDataTrailPanelAction } from 'app/features/trails/dashboardIntegration'; @@ -23,7 +22,7 @@ import { getDashboardSceneFor, getPanelIdForVizPanel, getQueryRunnerFor } from ' import { DashboardScene } from './DashboardScene'; import { LibraryVizPanel } from './LibraryVizPanel'; -import { VizPanelLinks } from './PanelLinks'; +import { VizPanelLinks, VizPanelLinksMenu } from './PanelLinks'; /** * Behavior is called when VizPanelMenu is activated (ie when it's opened). @@ -217,29 +216,39 @@ function getInspectMenuItem( /** * Behavior is called when VizPanelLinksMenu is activated (when it's opened). */ -export function getPanelLinksBehavior(panel: PanelModel) { - return (panelLinksMenu: VizPanelLinks) => { - const interpolate: InterpolateFunction = (v, scopedVars) => { - return sceneGraph.interpolate(panelLinksMenu, v, scopedVars); - }; +export function panelLinksBehavior(panelLinksMenu: VizPanelLinksMenu) { + if (!(panelLinksMenu.parent instanceof VizPanelLinks)) { + throw new Error('parent of VizPanelLinksMenu must be VizPanelLinks'); + } + const panel = panelLinksMenu.parent.parent; - const linkSupplier = getPanelLinksSupplier(panel, interpolate); + if (!(panel instanceof VizPanel)) { + throw new Error('parent of VizPanelLinks must be VizPanel'); + } - if (!linkSupplier) { - return; - } + panelLinksMenu.setState({ links: getPanelLinks(panel) }); +} - const panelLinks = linkSupplier && linkSupplier.getLinks(interpolate); - - const links = panelLinks.map((panelLink) => ({ - ...panelLink, - onClick: (e: any, origin: any) => { - DashboardInteractions.panelLinkClicked({ has_multiple_links: panelLinks.length > 1 }); - panelLink.onClick?.(e, origin); - }, - })); - panelLinksMenu.setState({ links }); +export function getPanelLinks(panel: VizPanel) { + const interpolate: InterpolateFunction = (v, scopedVars) => { + return sceneGraph.interpolate(panel, v, scopedVars); }; + + const linkSupplier = getScenePanelLinksSupplier(panel, interpolate); + + if (!linkSupplier) { + return []; + } + + const panelLinks = linkSupplier.getLinks(interpolate); + + return panelLinks.map((panelLink) => ({ + ...panelLink, + onClick: (e: any, origin: any) => { + DashboardInteractions.panelLinkClicked({ has_multiple_links: panelLinks.length > 1 }); + panelLink.onClick?.(e, origin); + }, + })); } function createExtensionContext(panel: VizPanel, dashboard: DashboardScene): PluginExtensionPanelContext { diff --git a/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap b/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap index cd2f371a21a..312b0161df7 100644 --- a/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap +++ b/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap @@ -110,6 +110,7 @@ exports[`transformSceneToSaveModel Given a scene with rows Should transform back "y": 1, }, "id": 15, + "links": [], "options": { "code": { "language": "plaintext", @@ -164,6 +165,7 @@ exports[`transformSceneToSaveModel Given a scene with rows Should transform back "y": 26, }, "id": 30, + "links": [], "options": { "code": { "language": "plaintext", @@ -376,6 +378,7 @@ exports[`transformSceneToSaveModel Given a simple scene with custom settings Sho "y": 0, }, "id": 28, + "links": [], "options": { "legend": { "calcs": [], @@ -434,6 +437,7 @@ exports[`transformSceneToSaveModel Given a simple scene with custom settings Sho "y": 9, }, "id": 29, + "links": [], "options": {}, "targets": [ { @@ -464,6 +468,7 @@ exports[`transformSceneToSaveModel Given a simple scene with custom settings Sho "y": 9, }, "id": 25, + "links": [], "options": { "code": { "language": "plaintext", @@ -692,6 +697,7 @@ exports[`transformSceneToSaveModel Given a simple scene with variables Should tr "y": 0, }, "id": 28, + "links": [], "options": { "legend": { "calcs": [], @@ -750,6 +756,7 @@ exports[`transformSceneToSaveModel Given a simple scene with variables Should tr "y": 9, }, "id": 29, + "links": [], "options": {}, "targets": [ { @@ -780,6 +787,7 @@ exports[`transformSceneToSaveModel Given a simple scene with variables Should tr "y": 9, }, "id": 25, + "links": [], "options": { "code": { "language": "plaintext", diff --git a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts index 4817713fee9..6a1dae0679a 100644 --- a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts +++ b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts @@ -40,7 +40,7 @@ import { registerDashboardMacro } from '../scene/DashboardMacro'; import { DashboardScene } from '../scene/DashboardScene'; import { LibraryVizPanel } from '../scene/LibraryVizPanel'; import { VizPanelLinks, VizPanelLinksMenu } from '../scene/PanelLinks'; -import { getPanelLinksBehavior, panelMenuBehavior } from '../scene/PanelMenuBehavior'; +import { panelLinksBehavior, panelMenuBehavior } from '../scene/PanelMenuBehavior'; import { PanelNotices } from '../scene/PanelNotices'; import { PanelRepeaterGridItem } from '../scene/PanelRepeaterGridItem'; import { PanelTimeRange } from '../scene/PanelTimeRange'; @@ -409,16 +409,14 @@ export function buildGridItemForLibPanel(panel: PanelModel) { } export function buildGridItemForPanel(panel: PanelModel): SceneGridItemLike { - const hasPanelLinks = panel.links && panel.links.length > 0; const titleItems: SceneObject[] = []; - let panelLinks; - if (hasPanelLinks) { - panelLinks = new VizPanelLinks({ - menu: new VizPanelLinksMenu({ $behaviors: [getPanelLinksBehavior(panel)] }), - }); - titleItems.push(panelLinks); - } + titleItems.push( + new VizPanelLinks({ + rawLinks: panel.links, + menu: new VizPanelLinksMenu({ $behaviors: [panelLinksBehavior] }), + }) + ); titleItems.push(new PanelNotices()); diff --git a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts index f64ee741f18..86b675a6f81 100644 --- a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts +++ b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts @@ -277,6 +277,39 @@ describe('transformSceneToSaveModel', () => { expect(saveModel.gridPos?.w).toBe(12); expect(saveModel.gridPos?.h).toBe(8); }); + it('Given panel with links', () => { + const gridItem = buildGridItemFromPanelSchema({ + title: '', + type: 'text-plugin-34', + gridPos: { x: 1, y: 2, w: 12, h: 8 }, + links: [ + // @ts-expect-error Panel link is wrongly typed as DashboardLink + { + title: 'Link 1', + url: 'http://some.test.link1', + }, + // @ts-expect-error Panel link is wrongly typed as DashboardLink + { + targetBlank: true, + title: 'Link 2', + url: 'http://some.test.link2', + }, + ], + }); + + const saveModel = gridItemToPanel(gridItem); + expect(saveModel.links).toEqual([ + { + title: 'Link 1', + url: 'http://some.test.link1', + }, + { + targetBlank: true, + title: 'Link 2', + url: 'http://some.test.link2', + }, + ]); + }); }); describe('Library panels', () => { diff --git a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts index 0b773c401b1..8639b905295 100644 --- a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts +++ b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts @@ -17,6 +17,7 @@ import { import { AnnotationQuery, Dashboard, + DashboardLink, DataTransformerConfig, defaultDashboard, defaultTimePickerConfig, @@ -37,6 +38,7 @@ import { LibraryVizPanel } from '../scene/LibraryVizPanel'; import { PanelRepeaterGridItem } from '../scene/PanelRepeaterGridItem'; import { PanelTimeRange } from '../scene/PanelTimeRange'; import { RowRepeaterBehavior } from '../scene/RowRepeaterBehavior'; +import { dashboardSceneGraph } from '../utils/dashboardSceneGraph'; import { getPanelIdForVizPanel } from '../utils/utils'; import { GRAFANA_DATASOURCE_REF } from './const'; @@ -223,6 +225,9 @@ export function gridItemToPanel(gridItem: SceneGridItemLike, isSnapshot = false) panel.repeatDirection = gridItem.getRepeatDirection(); } + const panelLinks = dashboardSceneGraph.getPanelLinks(vizPanel); + panel.links = (panelLinks.state.rawLinks as DashboardLink[]) ?? []; + return panel; } diff --git a/public/app/features/dashboard-scene/utils/dashboardSceneGraph.test.ts b/public/app/features/dashboard-scene/utils/dashboardSceneGraph.test.ts index 44f045df7a8..8b4c65ab284 100644 --- a/public/app/features/dashboard-scene/utils/dashboardSceneGraph.test.ts +++ b/public/app/features/dashboard-scene/utils/dashboardSceneGraph.test.ts @@ -11,8 +11,10 @@ import { import { DashboardControls } from '../scene/DashboardControls'; import { DashboardLinksControls } from '../scene/DashboardLinksControls'; import { DashboardScene, DashboardSceneState } from '../scene/DashboardScene'; +import { VizPanelLinks, VizPanelLinksMenu } from '../scene/PanelLinks'; import { dashboardSceneGraph } from './dashboardSceneGraph'; +import { findVizPanelByKey } from './utils'; describe('dashboardSceneGraph', () => { describe('getTimePicker', () => { @@ -75,6 +77,20 @@ describe('dashboardSceneGraph', () => { expect(dashboardControls).not.toBeNull(); }); }); + + describe('getPanelLinks', () => { + it('should throw if no links object defined', () => { + const scene = buildTestScene(); + const panelWithNoLinks = findVizPanelByKey(scene, 'panel-1')!; + expect(() => dashboardSceneGraph.getPanelLinks(panelWithNoLinks)).toThrow(); + }); + + it('should resolve VizPanelLinks object', () => { + const scene = buildTestScene(); + const panelWithNoLinks = findVizPanelByKey(scene, 'panel-with-links')!; + expect(dashboardSceneGraph.getPanelLinks(panelWithNoLinks)).toBeInstanceOf(VizPanelLinks); + }); + }); }); function buildTestScene(overrides?: Partial) { @@ -121,6 +137,15 @@ function buildTestScene(overrides?: Partial) { $data: new SceneQueryRunner({ key: 'data-query-runner2', queries: [{ refId: 'A' }] }), }), }), + new SceneGridItem({ + body: new VizPanel({ + title: 'Panel B', + key: 'panel-with-links', + pluginId: 'table', + $data: new SceneQueryRunner({ key: 'data-query-runner3', queries: [{ refId: 'A' }] }), + titleItems: [new VizPanelLinks({ menu: new VizPanelLinksMenu({}) })], + }), + }), ], }), ...overrides, diff --git a/public/app/features/dashboard-scene/utils/dashboardSceneGraph.ts b/public/app/features/dashboard-scene/utils/dashboardSceneGraph.ts index 666b764b001..641b21c7f3b 100644 --- a/public/app/features/dashboard-scene/utils/dashboardSceneGraph.ts +++ b/public/app/features/dashboard-scene/utils/dashboardSceneGraph.ts @@ -1,7 +1,8 @@ -import { SceneTimePicker, SceneRefreshPicker } from '@grafana/scenes'; +import { SceneTimePicker, SceneRefreshPicker, VizPanel } from '@grafana/scenes'; import { DashboardControls } from '../scene/DashboardControls'; import { DashboardScene } from '../scene/DashboardScene'; +import { VizPanelLinks } from '../scene/PanelLinks'; function getTimePicker(scene: DashboardScene) { const dashboardControls = getDashboardControls(scene); @@ -36,8 +37,21 @@ function getDashboardControls(scene: DashboardScene) { return null; } +function getPanelLinks(panel: VizPanel) { + if ( + panel.state.titleItems && + Array.isArray(panel.state.titleItems) && + panel.state.titleItems[0] instanceof VizPanelLinks + ) { + return panel.state.titleItems[0]; + } + + throw new Error('VizPanelLinks links not found'); +} + export const dashboardSceneGraph = { getTimePicker, getRefreshPicker, getDashboardControls, + getPanelLinks, }; diff --git a/public/app/features/dashboard-scene/utils/test-utils.ts b/public/app/features/dashboard-scene/utils/test-utils.ts index aa8a8afe29f..9e6b18d0f97 100644 --- a/public/app/features/dashboard-scene/utils/test-utils.ts +++ b/public/app/features/dashboard-scene/utils/test-utils.ts @@ -15,6 +15,7 @@ import { DashboardLoaderSrv, setDashboardLoaderSrv } from 'app/features/dashboar import { ALL_VARIABLE_TEXT, ALL_VARIABLE_VALUE } from 'app/features/variables/constants'; import { DashboardDTO } from 'app/types'; +import { VizPanelLinks, VizPanelLinksMenu } from '../scene/PanelLinks'; import { PanelRepeaterGridItem, RepeatDirection } from '../scene/PanelRepeaterGridItem'; import { RowRepeaterBehavior } from '../scene/RowRepeaterBehavior'; @@ -120,7 +121,11 @@ export function buildPanelRepeaterScene(options: SceneOptions) { y: 0, width: 10, height: 10, - body: new VizPanel({ title: 'Panel $server', pluginId: 'timeseries' }), + body: new VizPanel({ + title: 'Panel $server', + pluginId: 'timeseries', + titleItems: [new VizPanelLinks({ menu: new VizPanelLinksMenu({}) })], + }), }); const rowChildren = defaults.usePanelRepeater ? repeater : gridItem; diff --git a/public/app/features/dashboard/components/PanelEditor/getPanelFrameOptions.tsx b/public/app/features/dashboard/components/PanelEditor/getPanelFrameOptions.tsx index b093470a8d8..d24b39c46ab 100644 --- a/public/app/features/dashboard/components/PanelEditor/getPanelFrameOptions.tsx +++ b/public/app/features/dashboard/components/PanelEditor/getPanelFrameOptions.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { config } from '@grafana/runtime'; import { VizPanel } from '@grafana/scenes'; import { DataLinksInlineEditor, Input, RadioButtonGroup, Select, Switch, TextArea } from '@grafana/ui'; +import { dashboardSceneGraph } from 'app/features/dashboard-scene/utils/dashboardSceneGraph'; import { getPanelLinksVariableSuggestions } from 'app/features/panel/panellinks/link_srv'; import { GenAIPanelDescriptionButton } from '../GenAI/GenAIPanelDescriptionButton'; @@ -180,6 +181,9 @@ export function getPanelFrameCategory2(panel: VizPanel): OptionsPaneCategoryDesc isOpenDefault: true, }); + const panelLinksObject = dashboardSceneGraph.getPanelLinks(panel); + const links = panelLinksObject.state.rawLinks; + return descriptor .addItem( new OptionsPaneItemDescriptor({ @@ -234,29 +238,31 @@ export function getPanelFrameCategory2(panel: VizPanel): OptionsPaneCategoryDesc ); }, }) + ) + .addCategory( + new OptionsPaneCategoryDescriptor({ + title: 'Panel links', + id: 'Panel links', + isOpenDefault: false, + itemsCount: links?.length, + }).addItem( + new OptionsPaneItemDescriptor({ + title: 'Panel links', + render: function renderLinks() { + const { rawLinks: links } = panelLinksObject.useState(); + return ( + panelLinksObject.setState({ rawLinks: links })} + getSuggestions={getPanelLinksVariableSuggestions} + data={[]} + /> + ); + }, + }) + ) ); - // .addCategory( - // new OptionsPaneCategoryDescriptor({ - // title: 'Panel links', - // id: 'Panel links', - // isOpenDefault: false, - // itemsCount: panel.state.links?.length, - // }).addItem( - // new OptionsPaneItemDescriptor({ - // title: 'Panel links', - // render: function renderLinks() { - // return ( - // onPanelConfigChange('links', links)} - // getSuggestions={getPanelLinksVariableSuggestions} - // data={[]} - // /> - // ); - // }, - // }) - // ) - // ) + // // .addCategory( // new OptionsPaneCategoryDescriptor({ // title: 'Repeat options', diff --git a/public/app/features/panel/panellinks/linkSuppliers.ts b/public/app/features/panel/panellinks/linkSuppliers.ts index 0fa5cf47fb0..4924278d01f 100644 --- a/public/app/features/panel/panellinks/linkSuppliers.ts +++ b/public/app/features/panel/panellinks/linkSuppliers.ts @@ -11,7 +11,9 @@ import { ScopedVar, ScopedVars, } from '@grafana/data'; +import { VizPanel } from '@grafana/scenes'; import { PanelModel } from 'app/features/dashboard/state/PanelModel'; +import { dashboardSceneGraph } from 'app/features/dashboard-scene/utils/dashboardSceneGraph'; import { getLinkSrv } from './link_srv'; @@ -157,3 +159,22 @@ export const getPanelLinksSupplier = ( }, }; }; + +export const getScenePanelLinksSupplier = ( + panel: VizPanel, + replaceVariables: InterpolateFunction +): LinkModelSupplier | undefined => { + const links = dashboardSceneGraph.getPanelLinks(panel).state.rawLinks; + + if (!links || links.length === 0) { + return undefined; + } + + return { + getLinks: () => { + return links.map((link) => { + return getLinkSrv().getDataLinkUIModel(link, replaceVariables, panel); + }); + }, + }; +}; From 564a1d32b7470aabec19d13175f14c3838e681e9 Mon Sep 17 00:00:00 2001 From: Fabrizio <135109076+fabrizio-grafana@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:44:22 +0100 Subject: [PATCH 02/84] Add Python as dependency (#80979) --- .drone.yml | 13 ++++++++++++- scripts/drone/steps/lib.star | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 86a0013f759..9434baae52d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -123,6 +123,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -224,6 +225,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -535,6 +537,7 @@ steps: image: golang:1.21.6-alpine name: wire-install - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1044,6 +1047,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1403,6 +1407,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1479,6 +1484,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1537,6 +1543,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1799,6 +1806,7 @@ steps: image: golang:1.21.6-alpine name: wire-install - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -2714,6 +2722,7 @@ steps: image: golang:1.21.6-alpine name: compile-build-cmd - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -2987,6 +2996,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -3410,6 +3420,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: + - apt-get update -yq && apt-get install python -y - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -4777,6 +4788,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 0e34c95370617ee9f721421913cbe1fe103c117e2912ac589953298246fd2012 +hmac: d61ada44fb898f089416a8f2afcea19481065e311a771ce4b08e03f2d13b4853 ... diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 091380d3aee..81937e7a826 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -35,6 +35,7 @@ def yarn_install_step(): "name": "yarn-install", "image": images["node"], "commands": [ + "apt-get update -yq && apt-get install python -y", "yarn install --immutable", ], "depends_on": [], From e0054c06fe75437b84158abd07033bc927170853 Mon Sep 17 00:00:00 2001 From: Matias Chomicki Date: Wed, 31 Jan 2024 12:51:15 +0100 Subject: [PATCH 03/84] DashNav: add missing key (#81626) * DashNav: add missing key * DashNav: use more specific key --- .../app/features/dashboard/components/DashNav/DashNav.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/public/app/features/dashboard/components/DashNav/DashNav.tsx b/public/app/features/dashboard/components/DashNav/DashNav.tsx index 52ced1ac177..ffd013ebcf8 100644 --- a/public/app/features/dashboard/components/DashNav/DashNav.tsx +++ b/public/app/features/dashboard/components/DashNav/DashNav.tsx @@ -188,7 +188,13 @@ export const DashNav = React.memo((props) => { if (dashboard.meta.publicDashboardEnabled) { // TODO: This will be replaced with the new badge component. Color is required but gets override by css buttons.push( - + ); } From 42fe0fdf707e4b4b7e14cc3e99dc6425ab89843f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:05:38 +0000 Subject: [PATCH 04/84] Update dependency date-fns to v3 (#81625) * Update dependency date-fns to v3 * update imports --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Ashley Harrison --- package.json | 2 +- packages/grafana-data/package.json | 2 +- .../grafana-data/src/datetime/durationutil.ts | 4 +--- packages/grafana-ui/package.json | 2 +- .../components/RefreshPicker/RefreshPicker.tsx | 2 +- yarn.lock | 18 ++++++++---------- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 6d93ef2078b..e5169bc4912 100644 --- a/package.json +++ b/package.json @@ -298,7 +298,7 @@ "d3-force": "3.0.0", "d3-scale-chromatic": "3.0.0", "dangerously-set-html-content": "1.0.9", - "date-fns": "2.30.0", + "date-fns": "3.3.1", "debounce-promise": "3.1.2", "diff": "^5.1.0", "emotion": "11.0.0", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index bb5f23dd78e..aacd8bc4b19 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -40,7 +40,7 @@ "@types/d3-interpolate": "^3.0.0", "@types/string-hash": "1.1.3", "d3-interpolate": "3.0.1", - "date-fns": "2.30.0", + "date-fns": "3.3.1", "dompurify": "^2.4.3", "eventemitter3": "5.0.1", "fast_array_intersect": "1.1.0", diff --git a/packages/grafana-data/src/datetime/durationutil.ts b/packages/grafana-data/src/datetime/durationutil.ts index c490554a2bc..c8049a641bb 100644 --- a/packages/grafana-data/src/datetime/durationutil.ts +++ b/packages/grafana-data/src/datetime/durationutil.ts @@ -1,6 +1,4 @@ -import { Duration, Interval, isAfter } from 'date-fns'; -import add from 'date-fns/add'; -import intervalToDuration from 'date-fns/intervalToDuration'; +import { add, Duration, intervalToDuration, Interval, isAfter } from 'date-fns'; const durationMap: { [key in Required]: string[] } = { years: ['y', 'Y', 'years'], diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 256f4125c48..b37a30177cd 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -65,7 +65,7 @@ "calculate-size": "1.1.1", "classnames": "2.5.1", "d3": "7.8.5", - "date-fns": "2.30.0", + "date-fns": "3.3.1", "hoist-non-react-statics": "3.3.2", "i18next": "^22.0.0", "i18next-browser-languagedetector": "^7.0.2", diff --git a/packages/grafana-ui/src/components/RefreshPicker/RefreshPicker.tsx b/packages/grafana-ui/src/components/RefreshPicker/RefreshPicker.tsx index 1ea9e8e2163..7420a8034e0 100644 --- a/packages/grafana-ui/src/components/RefreshPicker/RefreshPicker.tsx +++ b/packages/grafana-ui/src/components/RefreshPicker/RefreshPicker.tsx @@ -1,4 +1,4 @@ -import formatDuration from 'date-fns/formatDuration'; +import { formatDuration } from 'date-fns'; import React, { PureComponent } from 'react'; import { SelectableValue, parseDuration } from '@grafana/data'; diff --git a/yarn.lock b/yarn.lock index 9cc0287bc34..57474c76384 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1670,7 +1670,7 @@ __metadata: languageName: node linkType: hard -"@babel/runtime@npm:7.23.9, @babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.11.1, @babel/runtime@npm:^7.11.2, @babel/runtime@npm:^7.12.0, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.13, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.14.0, @babel/runtime@npm:^7.14.5, @babel/runtime@npm:^7.15.4, @babel/runtime@npm:^7.17.2, @babel/runtime@npm:^7.17.8, @babel/runtime@npm:^7.18.0, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.20.0, @babel/runtime@npm:^7.20.7, @babel/runtime@npm:^7.21.0, @babel/runtime@npm:^7.23.2, @babel/runtime@npm:^7.3.1, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.7.2, @babel/runtime@npm:^7.7.6, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2": +"@babel/runtime@npm:7.23.9, @babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.11.1, @babel/runtime@npm:^7.11.2, @babel/runtime@npm:^7.12.0, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.13, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.14.0, @babel/runtime@npm:^7.14.5, @babel/runtime@npm:^7.15.4, @babel/runtime@npm:^7.17.2, @babel/runtime@npm:^7.17.8, @babel/runtime@npm:^7.18.0, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.20.0, @babel/runtime@npm:^7.20.7, @babel/runtime@npm:^7.23.2, @babel/runtime@npm:^7.3.1, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.7.2, @babel/runtime@npm:^7.7.6, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2": version: 7.23.9 resolution: "@babel/runtime@npm:7.23.9" dependencies: @@ -3249,7 +3249,7 @@ __metadata: "@types/testing-library__jest-dom": "npm:5.14.9" "@types/tinycolor2": "npm:1.4.6" d3-interpolate: "npm:3.0.1" - date-fns: "npm:2.30.0" + date-fns: "npm:3.3.1" dompurify: "npm:^2.4.3" esbuild: "npm:0.18.12" eventemitter3: "npm:5.0.1" @@ -3803,7 +3803,7 @@ __metadata: css-loader: "npm:6.10.0" csstype: "npm:3.1.3" d3: "npm:7.8.5" - date-fns: "npm:2.30.0" + date-fns: "npm:3.3.1" esbuild: "npm:0.18.12" expose-loader: "npm:4.1.0" hoist-non-react-statics: "npm:3.3.2" @@ -14038,12 +14038,10 @@ __metadata: languageName: node linkType: hard -"date-fns@npm:2.30.0": - version: 2.30.0 - resolution: "date-fns@npm:2.30.0" - dependencies: - "@babel/runtime": "npm:^7.21.0" - checksum: 10/70b3e8ea7aaaaeaa2cd80bd889622a4bcb5d8028b4de9162cbcda359db06e16ff6e9309e54eead5341e71031818497f19aaf9839c87d1aba1e27bb4796e758a9 +"date-fns@npm:3.3.1": + version: 3.3.1 + resolution: "date-fns@npm:3.3.1" + checksum: 10/98231936765dfb6fc6897676319b500a06a39f051b2c3ecbdd541a07ce9b1344b770277b8bfb1049fb7a2f70bf365ac8e6f1e2bb452b10e1a8101d518ca7f95d languageName: node linkType: hard @@ -17549,7 +17547,7 @@ __metadata: d3-force: "npm:3.0.0" d3-scale-chromatic: "npm:3.0.0" dangerously-set-html-content: "npm:1.0.9" - date-fns: "npm:2.30.0" + date-fns: "npm:3.3.1" debounce-promise: "npm:3.1.2" diff: "npm:^5.1.0" emotion: "npm:11.0.0" From 0851c18b55aef3287966da671ec893a5ad280bf5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:09:58 +0000 Subject: [PATCH 05/84] Update dependency @grafana/experimental to v1.7.9 --- package.json | 2 +- .../grafana-o11y-ds-frontend/package.json | 2 +- packages/grafana-sql/package.json | 2 +- .../datasource/azuremonitor/package.json | 2 +- .../datasource/cloud-monitoring/package.json | 2 +- .../grafana-testdata-datasource/package.json | 2 +- .../app/plugins/datasource/tempo/package.json | 2 +- yarn.lock | 23 ++++++++++--------- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index e5169bc4912..9d797b77b77 100644 --- a/package.json +++ b/package.json @@ -236,7 +236,7 @@ "@grafana/aws-sdk": "0.3.1", "@grafana/data": "workspace:*", "@grafana/e2e-selectors": "workspace:*", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/faro-core": "^1.3.6", "@grafana/faro-web-sdk": "^1.3.6", "@grafana/flamegraph": "workspace:*", diff --git a/packages/grafana-o11y-ds-frontend/package.json b/packages/grafana-o11y-ds-frontend/package.json index a5745a25ada..db3f6df8f4a 100644 --- a/packages/grafana-o11y-ds-frontend/package.json +++ b/packages/grafana-o11y-ds-frontend/package.json @@ -20,7 +20,7 @@ "@emotion/css": "11.11.2", "@grafana/data": "workspace:*", "@grafana/e2e-selectors": "workspace:*", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/runtime": "workspace:*", "@grafana/schema": "workspace:*", "@grafana/ui": "workspace:*", diff --git a/packages/grafana-sql/package.json b/packages/grafana-sql/package.json index b818a471c9b..c5e632a078f 100644 --- a/packages/grafana-sql/package.json +++ b/packages/grafana-sql/package.json @@ -16,7 +16,7 @@ "dependencies": { "@emotion/css": "11.11.2", "@grafana/data": "workspace:*", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/runtime": "workspace:*", "@grafana/ui": "workspace:*", "@react-awesome-query-builder/ui": "6.4.2", diff --git a/public/app/plugins/datasource/azuremonitor/package.json b/public/app/plugins/datasource/azuremonitor/package.json index 9491d4a1151..740b0339a58 100644 --- a/public/app/plugins/datasource/azuremonitor/package.json +++ b/public/app/plugins/datasource/azuremonitor/package.json @@ -6,7 +6,7 @@ "dependencies": { "@emotion/css": "11.11.2", "@grafana/data": "10.4.0-pre", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/runtime": "10.4.0-pre", "@grafana/schema": "10.4.0-pre", "@grafana/ui": "10.4.0-pre", diff --git a/public/app/plugins/datasource/cloud-monitoring/package.json b/public/app/plugins/datasource/cloud-monitoring/package.json index db53e73883e..e64cea2a291 100644 --- a/public/app/plugins/datasource/cloud-monitoring/package.json +++ b/public/app/plugins/datasource/cloud-monitoring/package.json @@ -6,7 +6,7 @@ "dependencies": { "@emotion/css": "11.11.2", "@grafana/data": "10.4.0-pre", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/google-sdk": "0.1.2", "@grafana/runtime": "10.4.0-pre", "@grafana/schema": "10.4.0-pre", diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/package.json b/public/app/plugins/datasource/grafana-testdata-datasource/package.json index cf3b8405e31..95fb7b28974 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/package.json +++ b/public/app/plugins/datasource/grafana-testdata-datasource/package.json @@ -6,7 +6,7 @@ "dependencies": { "@emotion/css": "11.11.2", "@grafana/data": "10.4.0-pre", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/runtime": "10.4.0-pre", "@grafana/schema": "10.4.0-pre", "@grafana/ui": "10.4.0-pre", diff --git a/public/app/plugins/datasource/tempo/package.json b/public/app/plugins/datasource/tempo/package.json index 2df60379a68..72bb414a789 100644 --- a/public/app/plugins/datasource/tempo/package.json +++ b/public/app/plugins/datasource/tempo/package.json @@ -7,7 +7,7 @@ "@emotion/css": "11.11.2", "@grafana/data": "workspace:*", "@grafana/e2e-selectors": "workspace:*", - "@grafana/experimental": "1.7.8", + "@grafana/experimental": "1.7.9", "@grafana/lezer-logql": "0.2.2", "@grafana/lezer-traceql": "0.0.13", "@grafana/monaco-logql": "^0.0.7", diff --git a/yarn.lock b/yarn.lock index 57474c76384..425de62c433 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2939,7 +2939,7 @@ __metadata: "@emotion/css": "npm:11.11.2" "@grafana/data": "npm:10.4.0-pre" "@grafana/e2e-selectors": "npm:10.4.0-pre" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/plugin-configs": "npm:10.4.0-pre" "@grafana/runtime": "npm:10.4.0-pre" "@grafana/schema": "npm:10.4.0-pre" @@ -2979,7 +2979,7 @@ __metadata: "@emotion/css": "npm:11.11.2" "@grafana/data": "npm:10.4.0-pre" "@grafana/e2e-selectors": "npm:10.4.0-pre" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/google-sdk": "npm:0.1.2" "@grafana/plugin-configs": "npm:10.4.0-pre" "@grafana/runtime": "npm:10.4.0-pre" @@ -3063,7 +3063,7 @@ __metadata: "@emotion/css": "npm:11.11.2" "@grafana/data": "npm:10.4.0-pre" "@grafana/e2e-selectors": "npm:10.4.0-pre" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/plugin-configs": "npm:10.4.0-pre" "@grafana/runtime": "npm:10.4.0-pre" "@grafana/schema": "npm:10.4.0-pre" @@ -3147,7 +3147,7 @@ __metadata: "@emotion/css": "npm:11.11.2" "@grafana/data": "workspace:*" "@grafana/e2e-selectors": "workspace:*" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/lezer-logql": "npm:0.2.2" "@grafana/lezer-traceql": "npm:0.0.13" "@grafana/monaco-logql": "npm:^0.0.7" @@ -3412,9 +3412,9 @@ __metadata: languageName: node linkType: hard -"@grafana/experimental@npm:1.7.8": - version: 1.7.8 - resolution: "@grafana/experimental@npm:1.7.8" +"@grafana/experimental@npm:1.7.9": + version: 1.7.9 + resolution: "@grafana/experimental@npm:1.7.9" dependencies: "@types/uuid": "npm:^8.3.3" lodash: "npm:^4.17.21" @@ -3427,13 +3427,14 @@ __metadata: peerDependencies: "@emotion/css": 11.11.2 "@grafana/data": ^10.0.0 + "@grafana/e2e-selectors": ^10.0.0 "@grafana/runtime": ^10.0.0 "@grafana/ui": ^10.0.0 react: 17.0.2 react-dom: 17.0.2 react-select: ^5.2.1 rxjs: 7.8.0 - checksum: 10/6bcf4a04b07fb1a34f7fa5332c0ab10760675e153d6d855b49f3cfe9fdc1b63223162aa82bac569b30f10ad4b2c434058bc03d50c650f50f86ddad01de1f4cce + checksum: 10/4dfa3f4c76c09ec275429407b16d5776657ce2d232ca0b8e049ba820d5f45dc74041e4c1e5d2fa1e8ec80221e5181c716a4311bf8e355302b51ee52e7186fa4d languageName: node linkType: hard @@ -3549,7 +3550,7 @@ __metadata: "@emotion/css": "npm:11.11.2" "@grafana/data": "workspace:*" "@grafana/e2e-selectors": "workspace:*" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/runtime": "workspace:*" "@grafana/schema": "workspace:*" "@grafana/tsconfig": "npm:^1.2.0-rc1" @@ -3680,7 +3681,7 @@ __metadata: dependencies: "@emotion/css": "npm:11.11.2" "@grafana/data": "workspace:*" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/runtime": "workspace:*" "@grafana/tsconfig": "npm:^1.3.0-rc1" "@grafana/ui": "workspace:*" @@ -17403,7 +17404,7 @@ __metadata: "@grafana/e2e-selectors": "workspace:*" "@grafana/eslint-config": "npm:7.0.0" "@grafana/eslint-plugin": "link:./packages/grafana-eslint-rules" - "@grafana/experimental": "npm:1.7.8" + "@grafana/experimental": "npm:1.7.9" "@grafana/faro-core": "npm:^1.3.6" "@grafana/faro-web-sdk": "npm:^1.3.6" "@grafana/flamegraph": "workspace:*" From a9f17a3f247a104f5de69692d06d5c7b9715331c Mon Sep 17 00:00:00 2001 From: Ivan Ortega Alba Date: Wed, 31 Jan 2024 13:31:08 +0100 Subject: [PATCH 06/84] QueryVariable: Be able to edit the variable using scenes (#80847) --- .betterer.results | 3 - .../src/selectors/pages.ts | 4 +- .../variables/components/QueryEditor.tsx | 78 +++++ .../components/QueryVariableForm.test.tsx | 269 ++++++++++++++++ .../components/QueryVariableForm.tsx | 128 ++++++++ .../components/VariableSelectField.tsx | 4 +- .../editors/QueryVariableEditor.test.tsx | 288 ++++++++++++++++++ .../variables/editors/QueryVariableEditor.tsx | 78 ++++- .../settings/variables/utils.test.ts | 4 +- .../editor/LegacyVariableQueryEditor.tsx | 2 +- .../query/QueryVariableEditor.test.tsx | 3 +- .../variables/query/QueryVariableEditor.tsx | 161 +++------- .../query/QueryVariableRefreshSelect.tsx | 5 +- .../query/QueryVariableSortSelect.tsx | 6 +- 14 files changed, 888 insertions(+), 145 deletions(-) create mode 100644 public/app/features/dashboard-scene/settings/variables/components/QueryEditor.tsx create mode 100644 public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.test.tsx create mode 100644 public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.tsx create mode 100644 public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.test.tsx diff --git a/.betterer.results b/.betterer.results index ba53b876c21..1f976380ac4 100644 --- a/.betterer.results +++ b/.betterer.results @@ -4330,9 +4330,6 @@ exports[`better eslint`] = { "public/app/features/variables/datasource/actions.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], - "public/app/features/variables/editor/LegacyVariableQueryEditor.tsx:5381": [ - [0, 0, 0, "Use data-testid for E2E selectors instead of aria-label", "0"] - ], "public/app/features/variables/editor/VariableEditorContainer.tsx:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], [0, 0, 0, "Do not use any type assertions.", "1"] diff --git a/packages/grafana-e2e-selectors/src/selectors/pages.ts b/packages/grafana-e2e-selectors/src/selectors/pages.ts index 285b0447aa2..20bb47d9029 100644 --- a/packages/grafana-e2e-selectors/src/selectors/pages.ts +++ b/packages/grafana-e2e-selectors/src/selectors/pages.ts @@ -145,14 +145,14 @@ export const Pages = { applyButton: 'data-testid Variable editor Apply button', }, QueryVariable: { - queryOptionsDataSourceSelect: Components.DataSourcePicker.container, + queryOptionsDataSourceSelect: Components.DataSourcePicker.inputV2, queryOptionsRefreshSelect: 'Variable editor Form Query Refresh select', queryOptionsRefreshSelectV2: 'data-testid Variable editor Form Query Refresh select', queryOptionsRegExInput: 'Variable editor Form Query RegEx field', queryOptionsRegExInputV2: 'data-testid Variable editor Form Query RegEx field', queryOptionsSortSelect: 'Variable editor Form Query Sort select', queryOptionsSortSelectV2: 'data-testid Variable editor Form Query Sort select', - queryOptionsQueryInput: 'Variable editor Form Default Variable Query Editor textarea', + queryOptionsQueryInput: 'data-testid Variable editor Form Default Variable Query Editor textarea', valueGroupsTagsEnabledSwitch: 'Variable editor Form Query UseTags switch', valueGroupsTagsTagsQueryInput: 'Variable editor Form Query TagsQuery field', valueGroupsTagsTagsValuesQueryInput: 'Variable editor Form Query TagsValuesQuery field', diff --git a/public/app/features/dashboard-scene/settings/variables/components/QueryEditor.tsx b/public/app/features/dashboard-scene/settings/variables/components/QueryEditor.tsx new file mode 100644 index 00000000000..18436c85c2b --- /dev/null +++ b/public/app/features/dashboard-scene/settings/variables/components/QueryEditor.tsx @@ -0,0 +1,78 @@ +import React from 'react'; + +import { DataSourceApi, LoadingState, TimeRange } from '@grafana/data'; +import { getTemplateSrv } from '@grafana/runtime'; +import { QueryVariable } from '@grafana/scenes'; +import { Text, Box } from '@grafana/ui'; +import { isLegacyQueryEditor, isQueryEditor } from 'app/features/variables/guard'; +import { VariableQueryEditorType } from 'app/features/variables/types'; + +type VariableQueryType = QueryVariable['state']['query']; + +interface QueryEditorProps { + query: VariableQueryType; + datasource: DataSourceApi; + VariableQueryEditor: VariableQueryEditorType; + timeRange: TimeRange; + onLegacyQueryChange: (query: VariableQueryType, definition: string) => void; + onQueryChange: (query: VariableQueryType) => void; +} + +export function QueryEditor({ + query, + datasource, + VariableQueryEditor, + onLegacyQueryChange, + onQueryChange, + timeRange, +}: QueryEditorProps) { + let queryWithDefaults; + if (typeof query === 'string') { + queryWithDefaults = query || (datasource.variables?.getDefaultQuery?.() ?? ''); + } else { + queryWithDefaults = { + ...datasource.variables?.getDefaultQuery?.(), + ...query, + }; + } + + if (VariableQueryEditor && isLegacyQueryEditor(VariableQueryEditor, datasource)) { + return ( + + Query + + + + + ); + } + + if (VariableQueryEditor && isQueryEditor(VariableQueryEditor, datasource)) { + return ( + + Query + + {}} + data={{ series: [], state: LoadingState.Done, timeRange }} + range={timeRange} + onBlur={() => {}} + history={[]} + /> + + + ); + } + + return null; +} diff --git a/public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.test.tsx b/public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.test.tsx new file mode 100644 index 00000000000..8e37364ccc7 --- /dev/null +++ b/public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.test.tsx @@ -0,0 +1,269 @@ +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import React, { FormEvent } from 'react'; +import { of } from 'rxjs'; +import { MockDataSourceApi } from 'test/mocks/datasource_srv'; + +import { + LoadingState, + PanelData, + getDefaultTimeRange, + toDataFrame, + FieldType, + VariableSupportType, +} from '@grafana/data'; +import { selectors } from '@grafana/e2e-selectors'; +import { setRunRequest } from '@grafana/runtime'; +import { VariableRefresh, VariableSort } from '@grafana/schema'; +import { mockDataSource } from 'app/features/alerting/unified/mocks'; +import { LegacyVariableQueryEditor } from 'app/features/variables/editor/LegacyVariableQueryEditor'; + +import { QueryVariableEditorForm } from './QueryVariableForm'; + +const defaultDatasource = mockDataSource({ + name: 'Default Test Data Source', + type: 'test', +}); + +const promDatasource = mockDataSource({ + name: 'Prometheus', + type: 'prometheus', +}); + +jest.mock('@grafana/runtime/src/services/dataSourceSrv', () => ({ + ...jest.requireActual('@grafana/runtime/src/services/dataSourceSrv'), + getDataSourceSrv: () => ({ + get: async () => ({ + ...defaultDatasource, + variables: { + getType: () => VariableSupportType.Custom, + query: jest.fn(), + editor: jest.fn().mockImplementation(LegacyVariableQueryEditor), + }, + }), + getList: () => [defaultDatasource, promDatasource], + getInstanceSettings: () => ({ ...defaultDatasource }), + }), +})); + +const runRequestMock = jest.fn().mockReturnValue( + of({ + state: LoadingState.Done, + series: [ + toDataFrame({ + fields: [{ name: 'text', type: FieldType.string, values: ['val1', 'val2', 'val11'] }], + }), + ], + timeRange: getDefaultTimeRange(), + }) +); + +setRunRequest(runRequestMock); + +describe('QueryVariableEditorForm', () => { + const mockOnDataSourceChange = jest.fn(); + const mockOnQueryChange = jest.fn(); + const mockOnLegacyQueryChange = jest.fn(); + const mockOnRegExChange = jest.fn(); + const mockOnSortChange = jest.fn(); + const mockOnRefreshChange = jest.fn(); + const mockOnMultiChange = jest.fn(); + const mockOnIncludeAllChange = jest.fn(); + const mockOnAllValueChange = jest.fn(); + + const defaultProps = { + datasource: new MockDataSourceApi(promDatasource.name, undefined, promDatasource.meta), + onDataSourceChange: mockOnDataSourceChange, + query: 'my-query', + onQueryChange: mockOnQueryChange, + onLegacyQueryChange: mockOnLegacyQueryChange, + timeRange: getDefaultTimeRange(), + VariableQueryEditor: LegacyVariableQueryEditor, + regex: '.*', + onRegExChange: mockOnRegExChange, + sort: VariableSort.alphabeticalAsc, + onSortChange: mockOnSortChange, + refresh: VariableRefresh.onDashboardLoad, + onRefreshChange: mockOnRefreshChange, + isMulti: true, + onMultiChange: mockOnMultiChange, + includeAll: true, + onIncludeAllChange: mockOnIncludeAllChange, + allValue: 'custom all value', + onAllValueChange: mockOnAllValueChange, + }; + + function setup(props?: React.ComponentProps) { + return { + renderer: render(), + user: userEvent.setup(), + }; + } + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should render the component with initializing the components correctly', () => { + const { + renderer: { getByTestId, getByRole }, + } = setup(); + const dataSourcePicker = getByTestId(selectors.components.DataSourcePicker.container); + //const queryEditor = getByTestId('query-editor'); + const regexInput = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2 + ); + const sortSelect = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsSortSelectV2 + ); + const refreshSelect = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRefreshSelectV2 + ); + + const multiSwitch = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsMultiSwitch + ); + const includeAllSwitch = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsIncludeAllSwitch + ); + const allValueInput = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsCustomAllInput + ); + + expect(dataSourcePicker).toBeInTheDocument(); + expect(dataSourcePicker).toHaveTextContent('Default Test Data Source'); + expect(regexInput).toBeInTheDocument(); + expect(regexInput).toHaveValue('.*'); + expect(sortSelect).toBeInTheDocument(); + expect(sortSelect).toHaveTextContent('Alphabetical (asc)'); + expect(refreshSelect).toBeInTheDocument(); + expect(getByRole('radio', { name: 'On dashboard load' })).toBeChecked(); + expect(multiSwitch).toBeInTheDocument(); + expect(multiSwitch).toBeChecked(); + expect(includeAllSwitch).toBeInTheDocument(); + expect(includeAllSwitch).toBeChecked(); + expect(allValueInput).toBeInTheDocument(); + expect(allValueInput).toHaveValue('custom all value'); + }); + + it('should call onDataSourceChange when changing the datasource', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const dataSourcePicker = getByTestId(selectors.components.DataSourcePicker.inputV2); + await waitFor(async () => { + await userEvent.click(dataSourcePicker); // open the select + await userEvent.tab(); + }); + expect(mockOnDataSourceChange).toHaveBeenCalledTimes(1); + expect(mockOnDataSourceChange).toHaveBeenCalledWith(defaultDatasource); + }); + + it('should call onQueryChange when changing the query', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const queryEditor = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput + ); + + await waitFor(async () => { + await userEvent.type(queryEditor, '-new'); + await userEvent.tab(); + }); + + expect(mockOnLegacyQueryChange).toHaveBeenCalledTimes(1); + expect(mockOnLegacyQueryChange).toHaveBeenCalledWith('my-query-new', expect.anything()); + }); + + it('should call onRegExChange when changing the regex', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const regexInput = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2 + ); + await userEvent.type(regexInput, '{backspace}?'); + await userEvent.tab(); + expect(mockOnRegExChange).toHaveBeenCalledTimes(1); + expect( + ((mockOnRegExChange.mock.calls[0][0] as FormEvent).target as HTMLTextAreaElement).value + ).toBe('.?'); + }); + + it('should call onSortChange when changing the sort', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const sortSelect = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsSortSelectV2 + ); + await userEvent.click(sortSelect); // open the select + const anotherOption = await screen.getByText('Alphabetical (desc)'); + await userEvent.click(anotherOption); + + expect(mockOnSortChange).toHaveBeenCalledTimes(1); + expect(mockOnSortChange).toHaveBeenCalledWith( + expect.objectContaining({ value: VariableSort.alphabeticalDesc }), + expect.anything() + ); + }); + + it('should call onRefreshChange when changing the refresh', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const refreshSelect = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRefreshSelectV2 + ); + await userEvent.click(refreshSelect); // open the select + const anotherOption = await screen.getByText('On time range change'); + await userEvent.click(anotherOption); + + expect(mockOnRefreshChange).toHaveBeenCalledTimes(1); + expect(mockOnRefreshChange).toHaveBeenCalledWith(VariableRefresh.onTimeRangeChanged); + }); + + it('should call onMultiChange when changing the multi switch', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const multiSwitch = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsMultiSwitch + ); + await userEvent.click(multiSwitch); + expect(mockOnMultiChange).toHaveBeenCalledTimes(1); + expect( + (mockOnMultiChange.mock.calls[0][0] as FormEvent).target as HTMLInputElement + ).toBeChecked(); + }); + + it('should call onIncludeAllChange when changing the include all switch', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const includeAllSwitch = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsIncludeAllSwitch + ); + await userEvent.click(includeAllSwitch); + expect(mockOnIncludeAllChange).toHaveBeenCalledTimes(1); + expect( + (mockOnIncludeAllChange.mock.calls[0][0] as FormEvent).target as HTMLInputElement + ).toBeChecked(); + }); + + it('should call onAllValueChange when changing the all value', async () => { + const { + renderer: { getByTestId }, + } = setup(); + const allValueInput = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsCustomAllInput + ); + await userEvent.type(allValueInput, ' and another value'); + await userEvent.tab(); + expect(mockOnAllValueChange).toHaveBeenCalledTimes(1); + expect( + ((mockOnAllValueChange.mock.calls[0][0] as FormEvent).target as HTMLInputElement).value + ).toBe('custom all value and another value'); + }); +}); diff --git a/public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.tsx b/public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.tsx new file mode 100644 index 00000000000..0e7d86740cc --- /dev/null +++ b/public/app/features/dashboard-scene/settings/variables/components/QueryVariableForm.tsx @@ -0,0 +1,128 @@ +import React, { FormEvent } from 'react'; + +import { DataSourceApi, DataSourceInstanceSettings, SelectableValue, TimeRange } from '@grafana/data'; +import { selectors } from '@grafana/e2e-selectors'; +import { QueryVariable } from '@grafana/scenes'; +import { VariableRefresh, VariableSort } from '@grafana/schema'; +import { Field } from '@grafana/ui'; +import { QueryEditor } from 'app/features/dashboard-scene/settings/variables/components/QueryEditor'; +import { SelectionOptionsForm } from 'app/features/dashboard-scene/settings/variables/components/SelectionOptionsForm'; +import { DataSourcePicker } from 'app/features/datasources/components/picker/DataSourcePicker'; +import { QueryVariableRefreshSelect } from 'app/features/variables/query/QueryVariableRefreshSelect'; +import { QueryVariableSortSelect } from 'app/features/variables/query/QueryVariableSortSelect'; +import { VariableQueryEditorType } from 'app/features/variables/types'; + +import { VariableLegend } from './VariableLegend'; +import { VariableTextAreaField } from './VariableTextAreaField'; + +type VariableQueryType = QueryVariable['state']['query']; + +interface QueryVariableEditorFormProps { + datasource: DataSourceApi | undefined; + onDataSourceChange: (dsSettings: DataSourceInstanceSettings) => void; + query: VariableQueryType; + onQueryChange: (query: VariableQueryType) => void; + onLegacyQueryChange: (query: VariableQueryType, definition: string) => void; + VariableQueryEditor: VariableQueryEditorType | undefined; + timeRange: TimeRange; + regex: string | null; + onRegExChange: (event: FormEvent) => void; + sort: VariableSort; + onSortChange: (option: SelectableValue) => void; + refresh: VariableRefresh; + onRefreshChange: (option: VariableRefresh) => void; + isMulti: boolean; + onMultiChange: (event: FormEvent) => void; + includeAll: boolean; + onIncludeAllChange: (event: FormEvent) => void; + allValue: string; + onAllValueChange: (event: FormEvent) => void; +} + +export function QueryVariableEditorForm({ + datasource, + onDataSourceChange, + query, + onQueryChange, + onLegacyQueryChange, + VariableQueryEditor, + timeRange, + regex, + onRegExChange, + sort, + onSortChange, + refresh, + onRefreshChange, + isMulti, + onMultiChange, + includeAll, + onIncludeAllChange, + allValue, + onAllValueChange, +}: QueryVariableEditorFormProps) { + return ( + <> + Query options + + + + + {datasource && VariableQueryEditor && ( + + )} + + + Optional, if you want to extract part of a series name or metric node segment. +
+ Named capture groups can be used to separate the display text and value ( + + see examples + + ). + + } + placeholder="/.*-(?.*)-(?.*)-.*/" + onBlur={onRegExChange} + testId={selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2} + width={52} + /> + + + + + + Selection options + + + ); +} diff --git a/public/app/features/dashboard-scene/settings/variables/components/VariableSelectField.tsx b/public/app/features/dashboard-scene/settings/variables/components/VariableSelectField.tsx index 17f273dab46..9f0024a7ab1 100644 --- a/public/app/features/dashboard-scene/settings/variables/components/VariableSelectField.tsx +++ b/public/app/features/dashboard-scene/settings/variables/components/VariableSelectField.tsx @@ -1,5 +1,5 @@ import { css } from '@emotion/css'; -import React, { PropsWithChildren, ReactElement, useId } from 'react'; +import React, { PropsWithChildren, useId } from 'react'; import { GrafanaTheme2, SelectableValue } from '@grafana/data'; import { Field, Select, useStyles2 } from '@grafana/ui'; @@ -22,7 +22,7 @@ export function VariableSelectField({ onChange, testId, width, -}: PropsWithChildren>): ReactElement { +}: PropsWithChildren>) { const styles = useStyles2(getStyles); const uniqueId = useId(); const inputId = `variable-select-input-${name}-${uniqueId}`; diff --git a/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.test.tsx b/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.test.tsx new file mode 100644 index 00000000000..867ecb3e836 --- /dev/null +++ b/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.test.tsx @@ -0,0 +1,288 @@ +import { getByRole, render, screen, act, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import React from 'react'; +import { lastValueFrom, of } from 'rxjs'; + +import { + VariableSupportType, + PanelData, + LoadingState, + toDataFrame, + getDefaultTimeRange, + FieldType, +} from '@grafana/data'; +import { selectors } from '@grafana/e2e-selectors'; +import { setRunRequest } from '@grafana/runtime'; +import { QueryVariable } from '@grafana/scenes'; +import { VariableRefresh, VariableSort } from '@grafana/schema'; +import { mockDataSource } from 'app/features/alerting/unified/mocks'; +import { LegacyVariableQueryEditor } from 'app/features/variables/editor/LegacyVariableQueryEditor'; + +import { QueryVariableEditor } from './QueryVariableEditor'; + +const defaultDatasource = mockDataSource({ + name: 'Default Test Data Source', + type: 'test', +}); + +const promDatasource = mockDataSource({ + name: 'Prometheus', + type: 'prometheus', +}); + +jest.mock('@grafana/runtime/src/services/dataSourceSrv', () => ({ + ...jest.requireActual('@grafana/runtime/src/services/dataSourceSrv'), + getDataSourceSrv: () => ({ + get: async () => ({ + ...defaultDatasource, + variables: { + getType: () => VariableSupportType.Custom, + query: jest.fn(), + editor: jest.fn().mockImplementation(LegacyVariableQueryEditor), + }, + }), + getList: () => [defaultDatasource, promDatasource], + getInstanceSettings: () => ({ ...defaultDatasource }), + }), +})); + +const runRequestMock = jest.fn().mockReturnValue( + of({ + state: LoadingState.Done, + series: [ + toDataFrame({ + fields: [{ name: 'text', type: FieldType.string, values: ['val1', 'val2', 'val11'] }], + }), + ], + timeRange: getDefaultTimeRange(), + }) +); + +setRunRequest(runRequestMock); + +describe('QueryVariableEditor', () => { + const onRunQueryMock = jest.fn(); + + async function setup(props?: React.ComponentProps) { + const variable = new QueryVariable({ + datasource: { + uid: defaultDatasource.uid, + type: defaultDatasource.type, + }, + query: 'my-query', + regex: '.*', + sort: VariableSort.alphabeticalAsc, + refresh: VariableRefresh.onDashboardLoad, + isMulti: true, + includeAll: true, + allValue: 'custom all value', + }); + + return { + renderer: await act(() => { + return render(); + }), + variable, + user: userEvent.setup(), + }; + } + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should render the component with initializing the components correctly', async () => { + const { renderer } = await setup(); + const dataSourcePicker = renderer.getByTestId(selectors.components.DataSourcePicker.container); + const queryEditor = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput + ); + const regexInput = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2 + ); + const sortSelect = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsSortSelectV2 + ); + const refreshSelect = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRefreshSelectV2 + ); + + const multiSwitch = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsMultiSwitch + ); + const includeAllSwitch = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsIncludeAllSwitch + ); + const allValueInput = renderer.getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsCustomAllInput + ); + + expect(dataSourcePicker).toBeInTheDocument(); + expect(dataSourcePicker).toHaveTextContent('Default Test Data Source'); + expect(queryEditor).toBeInTheDocument(); + expect(queryEditor).toHaveValue('my-query'); + expect(regexInput).toBeInTheDocument(); + expect(regexInput).toHaveValue('.*'); + expect(sortSelect).toBeInTheDocument(); + expect(sortSelect).toHaveTextContent('Alphabetical (asc)'); + expect(refreshSelect).toBeInTheDocument(); + expect(getByRole(refreshSelect, 'radio', { name: 'On dashboard load' })).toBeChecked(); + expect(multiSwitch).toBeInTheDocument(); + expect(multiSwitch).toBeChecked(); + expect(includeAllSwitch).toBeInTheDocument(); + expect(includeAllSwitch).toBeChecked(); + expect(allValueInput).toBeInTheDocument(); + expect(allValueInput).toHaveValue('custom all value'); + }); + + it('should update variable state when changing the datasource', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const dataSourcePicker = getByTestId(selectors.components.DataSourcePicker.container).getElementsByTagName('input'); + + await waitFor(async () => { + await user.type(dataSourcePicker[0], 'm'); + await user.tab(); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.datasource).toEqual({ uid: 'mock-ds-2', type: 'test' }); + }); + + it('should update the variable state when changing the query', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const queryEditor = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput + ); + + await waitFor(async () => { + await user.type(queryEditor, '-new'); + await user.tab(); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.query).toEqual('my-query-new'); + expect(onRunQueryMock).toHaveBeenCalledTimes(1); + }); + + it('should update the variable state when changing the regex', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const regexInput = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2 + ); + + await waitFor(async () => { + await user.type(regexInput, '{backspace}?'); + await user.tab(); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.regex).toBe('.?'); + }); + + it('should update the variable state when changing the sort', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const sortSelect = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsSortSelectV2 + ); + + await waitFor(async () => { + await user.click(sortSelect); + const anotherOption = await screen.getByText('Alphabetical (desc)'); + await user.click(anotherOption); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.sort).toBe(VariableSort.alphabeticalDesc); + }); + + it('should update the variable state when changing the refresh', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const refreshSelect = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRefreshSelectV2 + ); + + await waitFor(async () => { + await user.click(refreshSelect); + const anotherOption = await screen.getByText('On time range change'); + await user.click(anotherOption); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.refresh).toBe(VariableRefresh.onTimeRangeChanged); + }); + + it('should update the variable state when changing the multi switch', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const multiSwitch = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsMultiSwitch + ); + + await waitFor(async () => { + await user.click(multiSwitch); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.isMulti).toBe(false); + }); + + it('should update the variable state when changing the include all switch', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const includeAllSwitch = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsIncludeAllSwitch + ); + + await waitFor(async () => { + await user.click(includeAllSwitch); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.includeAll).toBe(false); + }); + + it('should update the variable state when changing the all value', async () => { + const { + variable, + renderer: { getByTestId }, + user, + } = await setup(); + const allValueInput = getByTestId( + selectors.pages.Dashboard.Settings.Variables.Edit.General.selectionOptionsCustomAllInput + ); + + await waitFor(async () => { + await user.type(allValueInput, ' and another value'); + await user.tab(); + await lastValueFrom(variable.validateAndUpdate()); + }); + + expect(variable.state.allValue).toBe('custom all value and another value'); + }); +}); diff --git a/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.tsx b/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.tsx index 57cf0b39ef4..0c577d74b6b 100644 --- a/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.tsx +++ b/public/app/features/dashboard-scene/settings/variables/editors/QueryVariableEditor.tsx @@ -1,12 +1,80 @@ -import React from 'react'; +import React, { FormEvent } from 'react'; +import { useAsync } from 'react-use'; -import { QueryVariable } from '@grafana/scenes'; +import { SelectableValue, DataSourceInstanceSettings } from '@grafana/data'; +import { getDataSourceSrv } from '@grafana/runtime'; +import { QueryVariable, sceneGraph } from '@grafana/scenes'; +import { DataSourceRef, VariableRefresh, VariableSort } from '@grafana/schema'; +import { getVariableQueryEditor } from 'app/features/variables/editor/getVariableQueryEditor'; + +import { QueryVariableEditorForm } from '../components/QueryVariableForm'; interface QueryVariableEditorProps { variable: QueryVariable; - onChange: (variable: QueryVariable) => void; + onRunQuery: () => void; } +type VariableQueryType = QueryVariable['state']['query']; -export function QueryVariableEditor(props: QueryVariableEditorProps) { - return
QueryVariableEditor
; +export function QueryVariableEditor({ variable, onRunQuery }: QueryVariableEditorProps) { + const { datasource: datasourceRef, regex, sort, refresh, isMulti, includeAll, allValue, query } = variable.useState(); + const { value: timeRange } = sceneGraph.getTimeRange(variable).useState(); + + const { value: dsConfig } = useAsync(async () => { + const datasource = await getDataSourceSrv().get(datasourceRef ?? ''); + const VariableQueryEditor = await getVariableQueryEditor(datasource); + + return { datasource, VariableQueryEditor }; + }, [datasourceRef]); + const { datasource, VariableQueryEditor } = dsConfig ?? {}; + + const onRegExChange = (event: React.FormEvent) => { + variable.setState({ regex: event.currentTarget.value }); + }; + const onSortChange = (sort: SelectableValue) => { + variable.setState({ sort: sort.value }); + }; + const onRefreshChange = (refresh: VariableRefresh) => { + variable.setState({ refresh: refresh }); + }; + const onMultiChange = (event: FormEvent) => { + variable.setState({ isMulti: event.currentTarget.checked }); + }; + const onIncludeAllChange = (event: FormEvent) => { + variable.setState({ includeAll: event.currentTarget.checked }); + }; + const onAllValueChange = (event: FormEvent) => { + variable.setState({ allValue: event.currentTarget.value }); + }; + const onDataSourceChange = (dsInstanceSettings: DataSourceInstanceSettings) => { + const datasource: DataSourceRef = { uid: dsInstanceSettings.uid, type: dsInstanceSettings.type }; + variable.setState({ datasource }); + }; + const onQueryChange = (query: VariableQueryType) => { + variable.setState({ query }); + onRunQuery(); + }; + + return ( + + ); } diff --git a/public/app/features/dashboard-scene/settings/variables/utils.test.ts b/public/app/features/dashboard-scene/settings/variables/utils.test.ts index 6d43b34891a..9cdafc13411 100644 --- a/public/app/features/dashboard-scene/settings/variables/utils.test.ts +++ b/public/app/features/dashboard-scene/settings/variables/utils.test.ts @@ -115,7 +115,7 @@ describe('getVariableEditor', () => { }); it.each(Object.keys(EDITABLE_VARIABLES) as EditableVariableType[])( - 'should define an editor for every variable type', + 'should define an editor for variable type "%s"', (type) => { const editor = getVariableEditor(type); expect(editor).toBeDefined(); @@ -130,7 +130,7 @@ describe('getVariableEditor', () => { ['datasource', DataSourceVariableEditor], ['adhoc', AdHocFiltersVariableEditor], ['textbox', TextBoxVariableEditor], - ])('should return the correct editor for each variable type', (type, ExpectedVariableEditor) => { + ])('should return the correct editor for variable type "%s"', (type, ExpectedVariableEditor) => { expect(getVariableEditor(type as EditableVariableType)).toBe(ExpectedVariableEditor); }); }); diff --git a/public/app/features/variables/editor/LegacyVariableQueryEditor.tsx b/public/app/features/variables/editor/LegacyVariableQueryEditor.tsx index 069aed31fb0..2b5f3e24317 100644 --- a/public/app/features/variables/editor/LegacyVariableQueryEditor.tsx +++ b/public/app/features/variables/editor/LegacyVariableQueryEditor.tsx @@ -34,7 +34,7 @@ export const LegacyVariableQueryEditor = ({ onChange, query }: VariableQueryEdit onBlur={onBlur} placeholder="Metric name or tags query" required - aria-label={selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput} + data-testid={selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput} cols={52} className={styles.textarea} /> diff --git a/public/app/features/variables/query/QueryVariableEditor.test.tsx b/public/app/features/variables/query/QueryVariableEditor.test.tsx index 03a1884428e..264d53fd6a8 100644 --- a/public/app/features/variables/query/QueryVariableEditor.test.tsx +++ b/public/app/features/variables/query/QueryVariableEditor.test.tsx @@ -3,6 +3,7 @@ import userEvent from '@testing-library/user-event'; import React from 'react'; import { DataSourceApi, VariableSupportType } from '@grafana/data'; +import { selectors } from '@grafana/e2e-selectors'; import { mockDataSource } from 'app/features/alerting/unified/mocks'; import { DataSourceType } from 'app/features/alerting/unified/utils/datasource'; @@ -144,7 +145,7 @@ describe('QueryVariableEditor', () => { }); const getQueryField = () => - screen.getByRole('textbox', { name: /variable editor form default variable query editor textarea/i }); + screen.getByTestId(selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsQueryInput); const getRegExField = () => screen.getByLabelText(/Regex/); diff --git a/public/app/features/variables/query/QueryVariableEditor.tsx b/public/app/features/variables/query/QueryVariableEditor.tsx index 8cdea8c630d..1f4e0016815 100644 --- a/public/app/features/variables/query/QueryVariableEditor.tsx +++ b/public/app/features/variables/query/QueryVariableEditor.tsx @@ -1,28 +1,19 @@ import React, { FormEvent, PureComponent } from 'react'; import { connect, ConnectedProps } from 'react-redux'; -import { DataSourceInstanceSettings, getDataSourceRef, LoadingState, SelectableValue } from '@grafana/data'; -import { selectors } from '@grafana/e2e-selectors'; -import { getTemplateSrv } from '@grafana/runtime'; -import { Field, Text, Box } from '@grafana/ui'; -import { DataSourcePicker } from 'app/features/datasources/components/picker/DataSourcePicker'; +import { DataSourceInstanceSettings, getDataSourceRef, SelectableValue } from '@grafana/data'; +import { QueryVariableEditorForm } from 'app/features/dashboard-scene/settings/variables/components/QueryVariableForm'; import { StoreState } from '../../../types'; import { getTimeSrv } from '../../dashboard/services/TimeSrv'; -import { VariableLegend } from '../../dashboard-scene/settings/variables/components/VariableLegend'; -import { VariableTextAreaField } from '../../dashboard-scene/settings/variables/components/VariableTextAreaField'; -import { SelectionOptionsEditor } from '../editor/SelectionOptionsEditor'; import { initialVariableEditorState } from '../editor/reducer'; import { getQueryVariableEditorState } from '../editor/selectors'; -import { OnPropChangeArguments, VariableEditorProps } from '../editor/types'; -import { isLegacyQueryEditor, isQueryEditor } from '../guard'; +import { VariableEditorProps } from '../editor/types'; import { changeVariableMultiValue } from '../state/actions'; import { getVariablesState } from '../state/selectors'; -import { QueryVariableModel, VariableRefresh, VariableSort, VariableWithMultiSupport } from '../types'; +import { QueryVariableModel, VariableRefresh, VariableSort } from '../types'; import { toKeyedVariableIdentifier } from '../utils'; -import { QueryVariableRefreshSelect } from './QueryVariableRefreshSelect'; -import { QueryVariableSortSelect } from './QueryVariableSortSelect'; import { changeQueryVariableDataSource, changeQueryVariableQuery, initQueryVariableEditor } from './actions'; const mapStateToProps = (state: StoreState, ownProps: OwnProps) => { @@ -105,10 +96,6 @@ export class QueryVariableEditorUnConnected extends PureComponent } }; - onRegExChange = (event: FormEvent) => { - this.setState({ regex: event.currentTarget.value }); - }; - onRegExBlur = async (event: FormEvent) => { const regex = event.currentTarget.value; if (this.props.variable.regex !== regex) { @@ -124,11 +111,19 @@ export class QueryVariableEditorUnConnected extends PureComponent this.props.onPropChange({ propName: 'sort', propValue: option.value, updateOptions: true }); }; - onSelectionOptionsChange = async ({ propValue, propName }: OnPropChangeArguments) => { - this.props.onPropChange({ propName, propValue, updateOptions: true }); + onMultiChange = (event: FormEvent) => { + this.props.onPropChange({ propName: 'multi', propValue: event.currentTarget.checked }); }; - renderQueryEditor = () => { + onIncludeAllChange = (event: FormEvent) => { + this.props.onPropChange({ propName: 'includeAll', propValue: event.currentTarget.checked }); + }; + + onAllValueChange = (event: FormEvent) => { + this.props.onPropChange({ propName: 'allValue', propValue: event.currentTarget.value }); + }; + + render() { const { extended, variable } = this.props; if (!extended || !extended.dataSource || !extended.VariableQueryEditor) { @@ -137,112 +132,30 @@ export class QueryVariableEditorUnConnected extends PureComponent const datasource = extended.dataSource; const VariableQueryEditor = extended.VariableQueryEditor; + const timeRange = getTimeSrv().timeRange(); - let query = variable.query; - - if (typeof query === 'string') { - query = query || (datasource.variables?.getDefaultQuery?.() ?? ''); - } else { - query = { - ...datasource.variables?.getDefaultQuery?.(), - ...variable.query, - }; - } - - if (isLegacyQueryEditor(VariableQueryEditor, datasource)) { - return ( - - Query - - - - - ); - } - - const range = getTimeSrv().timeRange(); - - if (isQueryEditor(VariableQueryEditor, datasource)) { - return ( - - Query - - {}} - data={{ series: [], state: LoadingState.Done, timeRange: range }} - range={range} - onBlur={() => {}} - history={[]} - /> - - - ); - } - - return null; - }; - - render() { return ( - <> - Query options - - - - - {this.renderQueryEditor()} - - - Optional, if you want to extract part of a series name or metric node segment. -
- Named capture groups can be used to separate the display text and value ( - - see examples - - ). - - } - placeholder="/.*-(?.*)-(?.*)-.*/" - onChange={this.onRegExChange} - onBlur={this.onRegExBlur} - testId={selectors.pages.Dashboard.Settings.Variables.Edit.QueryVariable.queryOptionsRegExInputV2} - width={52} - /> - - - - - - Selection options - - + ); } } diff --git a/public/app/features/variables/query/QueryVariableRefreshSelect.tsx b/public/app/features/variables/query/QueryVariableRefreshSelect.tsx index 36c5500959d..88475bcc75e 100644 --- a/public/app/features/variables/query/QueryVariableRefreshSelect.tsx +++ b/public/app/features/variables/query/QueryVariableRefreshSelect.tsx @@ -7,6 +7,7 @@ import { useMediaQueryChange } from 'app/core/hooks/useMediaQueryChange'; interface Props { onChange: (option: VariableRefresh) => void; refresh: VariableRefresh; + testId?: string; } const REFRESH_OPTIONS = [ @@ -14,7 +15,7 @@ const REFRESH_OPTIONS = [ { label: 'On time range change', value: VariableRefresh.onTimeRangeChanged }, ]; -export function QueryVariableRefreshSelect({ onChange, refresh }: PropsWithChildren) { +export function QueryVariableRefreshSelect({ onChange, refresh, testId }: PropsWithChildren) { const theme = useTheme2(); const [isSmallScreen, setIsSmallScreen] = useState(false); @@ -31,7 +32,7 @@ export function QueryVariableRefreshSelect({ onChange, refresh }: PropsWithChild ); return ( - + ) => void; sort: VariableSort; + testId?: string; } const SORT_OPTIONS = [ @@ -23,7 +23,7 @@ const SORT_OPTIONS = [ { label: 'Natural (desc)', value: VariableSort.naturalDesc }, ]; -export function QueryVariableSortSelect({ onChange, sort }: PropsWithChildren) { +export function QueryVariableSortSelect({ onChange, sort, testId }: PropsWithChildren) { const value = useMemo(() => SORT_OPTIONS.find((o) => o.value === sort) ?? SORT_OPTIONS[0], [sort]); return ( @@ -33,7 +33,7 @@ export function QueryVariableSortSelect({ onChange, sort }: PropsWithChildren ); From c9bc937919d882f55876320aac4a2e3d2c3e04a1 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 31 Jan 2024 13:39:07 +0100 Subject: [PATCH 07/84] Loki: Limit running of samples based on query and time range (#81585) * Loki: Limit running of samples based on query and time range * Update --- .../components/LokiQueryBuilder.test.tsx | 100 +++++++++++++++++- .../components/LokiQueryBuilder.tsx | 15 ++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.test.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.test.tsx index 024a7cddd5d..3cec60ded95 100644 --- a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.test.tsx +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.test.tsx @@ -3,11 +3,12 @@ import userEvent from '@testing-library/user-event'; import React from 'react'; import { dateTime } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { createLokiDatasource } from '../../__mocks__/datasource'; import { LokiOperationId, LokiVisualQuery } from '../types'; -import { LokiQueryBuilder } from './LokiQueryBuilder'; +import { LokiQueryBuilder, TIME_SPAN_TO_TRIGGER_SAMPLES } from './LokiQueryBuilder'; import { EXPLAIN_LABEL_FILTER_CONTENT } from './LokiQueryBuilderExplained'; const MISSING_LABEL_FILTER_ERROR_MESSAGE = 'Select at least 1 label filter (label and value)'; @@ -40,6 +41,14 @@ const createDefaultProps = () => { }; describe('LokiQueryBuilder', () => { + const originalLokiQueryHints = config.featureToggles.lokiQueryHints; + beforeEach(() => { + config.featureToggles.lokiQueryHints = true; + }); + + afterEach(() => { + config.featureToggles.lokiQueryHints = originalLokiQueryHints; + }); it('tries to load labels when no labels are selected', async () => { const props = createDefaultProps(); props.datasource.getDataSamples = jest.fn().mockResolvedValue([]); @@ -134,6 +143,95 @@ describe('LokiQueryBuilder', () => { expect(screen.queryByText(EXPLAIN_LABEL_FILTER_CONTENT)).not.toBeInTheDocument(); }); }); + + it('re-runs sample query when query changes', async () => { + const query = { + labels: [{ label: 'foo', op: '=', value: 'bar' }], + operations: [{ id: LokiOperationId.LineContains, params: ['error'] }], + }; + const props = createDefaultProps(); + props.datasource.getDataSamples = jest.fn().mockResolvedValue([]); + + const { rerender } = render(); + rerender( + + ); + + await waitFor(() => { + expect(props.datasource.getDataSamples).toHaveBeenCalledTimes(2); + }); + }); + + it('does not re-run sample query when query does not change', async () => { + const query = { + labels: [{ label: 'foo', op: '=', value: 'bar' }], + operations: [{ id: LokiOperationId.LineContains, params: ['error'] }], + }; + const props = createDefaultProps(); + props.datasource.getDataSamples = jest.fn().mockResolvedValue([]); + + const { rerender } = render(); + rerender(); + + await waitFor(() => { + expect(props.datasource.getDataSamples).toHaveBeenCalledTimes(1); + }); + }); + + it('re-run sample query when time range changes over 5 minutes', async () => { + const query = { + labels: [{ label: 'foo', op: '=', value: 'bar' }], + operations: [{ id: LokiOperationId.LineContains, params: ['error'] }], + }; + const props = createDefaultProps(); + const updatedFrom = dateTime(props.timeRange.from.valueOf() + TIME_SPAN_TO_TRIGGER_SAMPLES + 1000); + const updatedTo = dateTime(props.timeRange.to.valueOf() + TIME_SPAN_TO_TRIGGER_SAMPLES + 1000); + const updatedTimeRange = { + from: updatedFrom, + to: updatedTo, + raw: { + from: updatedFrom, + to: updatedTo, + }, + }; + props.datasource.getDataSamples = jest.fn().mockResolvedValue([]); + + const { rerender } = render(); + rerender(); + + await waitFor(() => { + expect(props.datasource.getDataSamples).toHaveBeenCalledTimes(2); + }); + }); + + it('does not re-run sample query when time range changes less than 5 minutes', async () => { + const query = { + labels: [{ label: 'foo', op: '=', value: 'bar' }], + operations: [{ id: LokiOperationId.LineContains, params: ['error'] }], + }; + const props = createDefaultProps(); + const updatedFrom = dateTime(props.timeRange.from.valueOf() + TIME_SPAN_TO_TRIGGER_SAMPLES - 1000); + const updatedTo = dateTime(props.timeRange.to.valueOf() + TIME_SPAN_TO_TRIGGER_SAMPLES - 1000); + const updatedTimeRange = { + from: updatedFrom, + to: updatedTo, + raw: { + from: updatedFrom, + to: updatedTo, + }, + }; + props.datasource.getDataSamples = jest.fn().mockResolvedValue([]); + + const { rerender } = render(); + rerender(); + + await waitFor(() => { + expect(props.datasource.getDataSamples).toHaveBeenCalledTimes(1); + }); + }); }); const getSelectParent = (input: HTMLElement) => diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx index 6c10532c5d0..0261c8f7bdb 100644 --- a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx @@ -1,4 +1,6 @@ +import { isEqual } from 'lodash'; import React, { useEffect, useMemo, useState } from 'react'; +import { usePrevious } from 'react-use'; import { DataSourceApi, getDefaultTimeRange, LoadingState, PanelData, SelectableValue, TimeRange } from '@grafana/data'; import { @@ -29,6 +31,7 @@ import { LokiOperationId, LokiVisualQuery } from '../types'; import { EXPLAIN_LABEL_FILTER_CONTENT } from './LokiQueryBuilderExplained'; import { NestedQueryList } from './NestedQueryList'; +export const TIME_SPAN_TO_TRIGGER_SAMPLES = 5 * 60 * 1000; export interface Props { query: LokiVisualQuery; datasource: LokiDatasource; @@ -41,6 +44,8 @@ export const LokiQueryBuilder = React.memo( ({ datasource, query, onChange, onRunQuery, showExplain, timeRange }) => { const [sampleData, setSampleData] = useState(); const [highlightedOp, setHighlightedOp] = useState(undefined); + const prevQuery = usePrevious(query); + const prevTimeRange = usePrevious(timeRange); const onChangeLabels = (labels: QueryBuilderLabelFilter[]) => { onChange({ ...query, labels }); @@ -109,10 +114,16 @@ export const LokiQueryBuilder = React.memo( setSampleData(sampleData); }; - if (config.featureToggles.lokiQueryHints) { + const updateBasedOnChangedTimeRange = + prevTimeRange && + timeRange && + (Math.abs(timeRange.to.valueOf() - prevTimeRange.to.valueOf()) > TIME_SPAN_TO_TRIGGER_SAMPLES || + Math.abs(timeRange.from.valueOf() - prevTimeRange.from.valueOf()) > TIME_SPAN_TO_TRIGGER_SAMPLES); + const updateBasedOnChangedQuery = !isEqual(prevQuery, query); + if (config.featureToggles.lokiQueryHints && (updateBasedOnChangedTimeRange || updateBasedOnChangedQuery)) { onGetSampleData().catch(console.error); } - }, [datasource, query, timeRange]); + }, [datasource, query, timeRange, prevQuery, prevTimeRange]); const lang = { grammar: logqlGrammar, name: 'logql' }; return ( From 5c258b7a4bdb4db7b242994ed262997f43793872 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:32:50 +0000 Subject: [PATCH 08/84] Update dependency @grafana/scenes to v2.2.2 --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 425de62c433..b415bf5da76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3638,8 +3638,8 @@ __metadata: linkType: soft "@grafana/scenes@npm:^2.2.0": - version: 2.2.1 - resolution: "@grafana/scenes@npm:2.2.1" + version: 2.2.2 + resolution: "@grafana/scenes@npm:2.2.2" dependencies: "@grafana/e2e-selectors": "npm:10.0.2" react-grid-layout: "npm:1.3.4" @@ -3651,7 +3651,7 @@ __metadata: "@grafana/runtime": 10.0.3 "@grafana/schema": 10.0.3 "@grafana/ui": 10.0.3 - checksum: 10/3b260f2121a84934314020187435214c4616cd74ddc2b8b32993308c8719fd6d4b49631a17684a03cba0d01d3882d2eca6d318bee47244f3a0e246ae969e0e61 + checksum: 10/91f17fb1c60e4e355d9f1846412febd138aee27d4713bd76ccc4fa80a8811f7038862d3eb6a52fe721db014e343b265110786433d64d547c0b51ebdbefcff889 languageName: node linkType: hard From 0633f5501c4ccd553957d922b9f6e23d583042a6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 13:32:18 +0000 Subject: [PATCH 09/84] Update dependency dompurify to v3 (#81638) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/grafana-data/package.json | 4 ++-- yarn.lock | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index aacd8bc4b19..54922002175 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -41,7 +41,7 @@ "@types/string-hash": "1.1.3", "d3-interpolate": "3.0.1", "date-fns": "3.3.1", - "dompurify": "^2.4.3", + "dompurify": "^3.0.0", "eventemitter3": "5.0.1", "fast_array_intersect": "1.1.0", "history": "4.10.1", @@ -70,7 +70,7 @@ "@testing-library/jest-dom": "6.4.0", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", - "@types/dompurify": "^2", + "@types/dompurify": "^3.0.0", "@types/history": "4.7.11", "@types/jest": "29.5.11", "@types/jquery": "3.5.29", diff --git a/yarn.lock b/yarn.lock index b415bf5da76..6ec6b000a8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3235,7 +3235,7 @@ __metadata: "@testing-library/react": "npm:14.2.0" "@testing-library/user-event": "npm:14.5.2" "@types/d3-interpolate": "npm:^3.0.0" - "@types/dompurify": "npm:^2" + "@types/dompurify": "npm:^3.0.0" "@types/history": "npm:4.7.11" "@types/jest": "npm:29.5.11" "@types/jquery": "npm:3.5.29" @@ -3250,7 +3250,7 @@ __metadata: "@types/tinycolor2": "npm:1.4.6" d3-interpolate: "npm:3.0.1" date-fns: "npm:3.3.1" - dompurify: "npm:^2.4.3" + dompurify: "npm:^3.0.0" esbuild: "npm:0.18.12" eventemitter3: "npm:5.0.1" fast_array_intersect: "npm:1.1.0" @@ -8484,12 +8484,12 @@ __metadata: languageName: node linkType: hard -"@types/dompurify@npm:^2": - version: 2.4.0 - resolution: "@types/dompurify@npm:2.4.0" +"@types/dompurify@npm:^3.0.0": + version: 3.0.5 + resolution: "@types/dompurify@npm:3.0.5" dependencies: "@types/trusted-types": "npm:*" - checksum: 10/02771f206873d2b32fcd5c3c20a64a9f089d8376aaef9b7ebe07e0e78127e9518374730de0df39b7ee15ab90ac0bf4df7350c0f7fce4717a3342e1a4a5bd15ed + checksum: 10/e544b3ce53c41215cabff3d89256ff707c7ee8e0c9a1b5034b22014725d288b16e6942cdcdeeb4221c578c3421a6a4721aa0676431f55d7abd18c07368855c5e languageName: node linkType: hard @@ -14645,13 +14645,20 @@ __metadata: languageName: node linkType: hard -"dompurify@npm:^2.2.0, dompurify@npm:^2.4.3": +"dompurify@npm:^2.2.0": version: 2.4.5 resolution: "dompurify@npm:2.4.5" checksum: 10/d764c2ff126b3749dad35bc34eed40f51141d7dfd620e938c92f08d68c32beeb259d06abadeee91f6e2a8c8737ce670e2124ac9a257ba3bcdc666598cebcde01 languageName: node linkType: hard +"dompurify@npm:^3.0.0": + version: 3.0.8 + resolution: "dompurify@npm:3.0.8" + checksum: 10/671fa18bd4bcb1a6ff2e59ecf919f807615b551e7add8834b27751d4e0f3d754a67725482d1efdd259317cadcaaccb72a8afc3aba829ac59730e760041591a1a + languageName: node + linkType: hard + "domutils@npm:^2.5.2, domutils@npm:^2.6.0": version: 2.8.0 resolution: "domutils@npm:2.8.0" From 607ec6c96a9a85eb9908fe85781913be46e6700b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Wed, 31 Jan 2024 15:02:29 +0100 Subject: [PATCH 10/84] CODEOWNERS: better root-file selection (#81633) --- .github/CODEOWNERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02a56c82a58..43b600a7345 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -344,9 +344,9 @@ /packages/grafana-sql/ @grafana/partner-datasources @grafana/oss-big-tent # root files, mostly frontend -.browserslistrc @grafana/frontend-ops -package.json @grafana/frontend-ops -tsconfig.json @grafana/frontend-ops +/.browserslistrc @grafana/frontend-ops +/package.json @grafana/frontend-ops +/tsconfig.json @grafana/frontend-ops /.editorconfig @grafana/frontend-ops /.eslintignore @grafana/frontend-ops /.gitattributes @grafana/frontend-ops @@ -356,7 +356,7 @@ tsconfig.json @grafana/frontend-ops /.yarn @grafana/frontend-ops /.yarnrc.yml @grafana/frontend-ops /yarn.lock @grafana/frontend-ops -lerna.json @grafana/frontend-ops +/lerna.json @grafana/frontend-ops /.prettierrc.js @grafana/frontend-ops /.eslintrc @grafana/frontend-ops /.vim @zoltanbedi @@ -367,9 +367,9 @@ lerna.json @grafana/frontend-ops /tools/ @grafana/frontend-ops /lefthook.yml @grafana/frontend-ops /lefthook.rc @grafana/frontend-ops -.husky/pre-commit @grafana/frontend-ops -cypress.config.js @grafana/grafana-frontend-platform -.levignore.js @grafana/plugins-platform-frontend +/.husky/pre-commit @grafana/frontend-ops +/cypress.config.js @grafana/grafana-frontend-platform +/.levignore.js @grafana/plugins-platform-frontend # public folder From a7f224987800db6d3234306c6cfe0ff0bfd4d4eb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 14:25:50 +0000 Subject: [PATCH 11/84] Update dependency react-zoom-pan-pinch to v3.4.0 (#81642) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6ec6b000a8a..940cb901f00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25851,12 +25851,12 @@ __metadata: linkType: hard "react-zoom-pan-pinch@npm:^3.3.0": - version: 3.3.0 - resolution: "react-zoom-pan-pinch@npm:3.3.0" + version: 3.4.0 + resolution: "react-zoom-pan-pinch@npm:3.4.0" peerDependencies: react: "*" react-dom: "*" - checksum: 10/56e102f603dc5d0dbcff2effe403a1f46b718bde0bbad395fc990db0ef48f25fefb6969fec465948e90471a8e8e702cddaaa93742f77075cb1f4f32b10c1ab43 + checksum: 10/720f3ce6b42bbbfbb56bb684d525d0824978198c0650a1d20c3fff52ea595635bea7c748cd02cf75c0833b2bab2971dc129aa6bc4e06fa212e3c962ac63312ce languageName: node linkType: hard From c8f47e0c54faccafe94008e45187951fe6a7243f Mon Sep 17 00:00:00 2001 From: Darren Janeczek <38694490+darrenjaneczek@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:27:44 -0500 Subject: [PATCH 12/84] datatrails: UI improvements (#81427) * fix: datatrails: limit width of metric description * fix: datatrails: use vertical radio list for long label list * fix: datatrails: reduce spacing between header items * fix: datatrails: reduce gap in lower toolbar * fix: change to use Select component for longer lists --- .../trails/ActionTabs/BreakdownScene.tsx | 30 +++++++++++++++---- .../trails/ActionTabs/MetricOverviewScene.tsx | 4 ++- public/app/features/trails/DataTrail.tsx | 2 +- public/app/features/trails/MetricScene.tsx | 2 +- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/public/app/features/trails/ActionTabs/BreakdownScene.tsx b/public/app/features/trails/ActionTabs/BreakdownScene.tsx index 114eaaeb1a6..beb75b4591d 100644 --- a/public/app/features/trails/ActionTabs/BreakdownScene.tsx +++ b/public/app/features/trails/ActionTabs/BreakdownScene.tsx @@ -19,7 +19,7 @@ import { SceneQueryRunner, VariableDependencyConfig, } from '@grafana/scenes'; -import { Button, Field, RadioButtonGroup, useStyles2 } from '@grafana/ui'; +import { Button, Field, Select, RadioButtonGroup, useStyles2 } from '@grafana/ui'; import { ALL_VARIABLE_VALUE } from 'app/features/variables/constants'; import { getAutoQueriesForMetric } from '../AutomaticMetricQueries/AutoQueryEngine'; @@ -109,7 +109,11 @@ export class BreakdownScene extends SceneObjectBase { this.setState(stateUpdate); } - public onChange = (value: string) => { + public onChange = (value?: string) => { + if (!value) { + return; + } + const variable = this.getVariable(); if (value === ALL_VARIABLE_VALUE) { @@ -125,13 +129,26 @@ export class BreakdownScene extends SceneObjectBase { const { labels, body, loading, value } = model.useState(); const styles = useStyles2(getStyles); + const useHorizontalLabelSelector = labels.length <= 6; + return (
{loading &&
Loading...
}
- - - + {!loading && ( + + {useHorizontalLabelSelector ? ( + + ) : ( +
- {searchTooStrictWarning} +
+ + + +
+ {showStatus}
); @@ -291,13 +370,11 @@ function getCardPanelFor(metric: string) { } // Computes the Levenshtein distance between two strings, twice, once for the first half and once for the whole string. -function sortRelatedMetrics(metricList: VariableValueOption[], metric: string) { - return metricList.sort((a, b) => { - const aValue = String(a.value); +function sortRelatedMetrics(metricList: string[], metric: string) { + return metricList.sort((aValue, bValue) => { const aSplit = aValue.split('_'); const aHalf = aSplit.slice(0, aSplit.length / 2).join('_'); - const bValue = String(b.value); const bSplit = bValue.split('_'); const bHalf = bSplit.slice(0, bSplit.length / 2).join('_'); @@ -324,7 +401,7 @@ function getStyles(theme: GrafanaTheme2) { gap: theme.spacing(2), marginBottom: theme.spacing(1), }), - alternateMessage: css({ + statusMessage: css({ fontStyle: 'italic', marginTop: theme.spacing(7), textAlign: 'center', From b4d363e8fe59d01fbed9e855bf20cad949095f21 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Jan 2024 18:09:32 +0300 Subject: [PATCH 14/84] Chore: Fix broken anonymous devices test (#81649) --- pkg/services/anonymous/anonimpl/impl_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/services/anonymous/anonimpl/impl_test.go b/pkg/services/anonymous/anonimpl/impl_test.go index 51e28febacb..e3c93db2ff2 100644 --- a/pkg/services/anonymous/anonimpl/impl_test.go +++ b/pkg/services/anonymous/anonimpl/impl_test.go @@ -206,6 +206,8 @@ func TestIntegrationDeviceService_SearchDevice(t *testing.T) { Query: "", Page: 1, Limit: 1, + From: fixedTime, + To: fixedTime.Add(1 * time.Hour), }, expectedCount: 1, }, @@ -227,6 +229,8 @@ func TestIntegrationDeviceService_SearchDevice(t *testing.T) { Query: "192.1", Page: 1, Limit: 50, + From: fixedTime, + To: fixedTime.Add(1 * time.Hour), }, expectedCount: 1, expectedDevice: &anonstore.Device{ From 1bcd597bc02d1b8631fce1a13fe9c60187451195 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Jan 2024 18:25:11 +0300 Subject: [PATCH 15/84] Nested folders: Improve performance of shared with me dashboards listing (#81590) * Nested folders: Improve performance of shared with me dashboards listing * Fix tests * Clean up guardian --- .../dashboards/service/dashboard_service.go | 32 ++++++++----------- pkg/services/folder/folderimpl/folder_test.go | 31 ++++-------------- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/pkg/services/dashboards/service/dashboard_service.go b/pkg/services/dashboards/service/dashboard_service.go index b61042c7424..12b459d478f 100644 --- a/pkg/services/dashboards/service/dashboard_service.go +++ b/pkg/services/dashboards/service/dashboard_service.go @@ -598,35 +598,29 @@ func (dr *DashboardServiceImpl) filterUserSharedDashboards(ctx context.Context, folderUIDs = append(folderUIDs, dashboard.FolderUID) } - dashFolders, err := dr.folderStore.GetFolders(ctx, user.GetOrgID(), folderUIDs) + // GetFolders return only folders available to user. So we can use is to check access. + userDashFolders, err := dr.folderService.GetFolders(ctx, folder.GetFoldersQuery{ + UIDs: folderUIDs, + OrgID: user.GetOrgID(), + SignedInUser: user, + }) if err != nil { return nil, folder.ErrInternal.Errorf("failed to fetch parent folders from store: %w", err) } + dashFoldersMap := make(map[string]*folder.Folder, 0) + for _, f := range userDashFolders { + dashFoldersMap[f.UID] = f + } + for _, dashboard := range userDashboards { // Filter out dashboards if user has access to parent folder if dashboard.FolderUID == "" { continue } - dashFolder, ok := dashFolders[dashboard.FolderUID] - if !ok { - dr.log.Error("failed to fetch folder by UID from store", "uid", dashboard.FolderUID) - continue - } - - g, err := guardian.NewByFolder(ctx, dashFolder, user.GetOrgID(), user) - if err != nil { - dr.log.Error("failed to check folder permissions", "folder uid", dashboard.FolderUID, "error", err) - continue - } - - canView, err := g.CanView() - if err != nil { - dr.log.Error("failed to fetch dashboard", "uid", dashboard.UID, "error", err) - continue - } - if !canView { + _, hasAccess := dashFoldersMap[dashboard.FolderUID] + if !hasAccess { filteredDashboards = append(filteredDashboards, dashboard) } } diff --git a/pkg/services/folder/folderimpl/folder_test.go b/pkg/services/folder/folderimpl/folder_test.go index 85ba55c66c0..f88ba017862 100644 --- a/pkg/services/folder/folderimpl/folder_test.go +++ b/pkg/services/folder/folderimpl/folder_test.go @@ -1293,7 +1293,7 @@ func TestIntegrationNestedFolderSharedWithMe(t *testing.T) { acmock.NewMockedPermissionsService(), dashboardPermissions, actest.FakeAccessControl{}, - foldertest.NewFakeService(), + serviceWithFlagOn, nil, ) require.NoError(t, err) @@ -1318,13 +1318,13 @@ func TestIntegrationNestedFolderSharedWithMe(t *testing.T) { SignedInUser: &signedInAdminUser, } + guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{ + CanSaveValue: true, + CanViewValue: true, + }) + t.Run("Should get folders shared with given user", func(t *testing.T) { depth := 3 - origNewGuardian := guardian.New - guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{ - CanSaveValue: true, - CanViewValue: true, - }) ancestorFoldersWithPermissions := CreateSubtreeInStore(t, nestedFolderStore, serviceWithFlagOn, depth, "withPermissions", createCmd) ancestorFoldersWithoutPermissions := CreateSubtreeInStore(t, nestedFolderStore, serviceWithFlagOn, depth, "withoutPermissions", createCmd) @@ -1338,17 +1338,6 @@ func TestIntegrationNestedFolderSharedWithMe(t *testing.T) { // nolint:staticcheck dash2 := insertTestDashboard(t, serviceWithFlagOn.dashboardStore, "dashboard in subfolder", orgID, subfolder.ID, subfolder.UID, "prod") - guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{ - CanSaveValue: true, - CanViewValue: true, - CanViewUIDs: []string{ - ancestorFoldersWithPermissions[0].UID, - ancestorFoldersWithPermissions[1].UID, - ancestorFoldersWithoutPermissions[1].UID, - dash1.UID, - dash2.UID, - }, - }) signedInUser.Permissions[orgID][dashboards.ActionFoldersRead] = []string{ dashboards.ScopeFoldersProvider.GetResourceScopeUID(ancestorFoldersWithPermissions[0].UID), // Add permission to the subfolder of folder with permission (to check deduplication) @@ -1390,7 +1379,7 @@ func TestIntegrationNestedFolderSharedWithMe(t *testing.T) { require.NotContains(t, sharedDashboardsUIDs, dash2.UID) t.Cleanup(func() { - guardian.New = origNewGuardian + //guardian.New = origNewGuardian toDelete := make([]string, 0, len(ancestorFoldersWithPermissions)+len(ancestorFoldersWithoutPermissions)) for _, ancestor := range append(ancestorFoldersWithPermissions, ancestorFoldersWithoutPermissions...) { toDelete = append(toDelete, ancestor.UID) @@ -1402,11 +1391,6 @@ func TestIntegrationNestedFolderSharedWithMe(t *testing.T) { t.Run("Should get org folders visible", func(t *testing.T) { depth := 3 - origNewGuardian := guardian.New - guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{ - CanSaveValue: true, - CanViewValue: true, - }) // create folder sctructure like this: // tree1-folder-0 @@ -1428,7 +1412,6 @@ func TestIntegrationNestedFolderSharedWithMe(t *testing.T) { } t.Cleanup(func() { - guardian.New = origNewGuardian toDelete := make([]string, 0, len(tree1)+len(tree2)) for _, f := range append(tree1, tree2...) { toDelete = append(toDelete, f.UID) From 91b2909c390a0466db3dda500c1c9e50b6784bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laura=20Fern=C3=A1ndez?= Date: Wed, 31 Jan 2024 17:04:34 +0100 Subject: [PATCH 16/84] ReturnToPrevious: add shadow to be consistent (#81652) --- .../components/AppChrome/ReturnToPrevious/ReturnToPrevious.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/core/components/AppChrome/ReturnToPrevious/ReturnToPrevious.tsx b/public/app/core/components/AppChrome/ReturnToPrevious/ReturnToPrevious.tsx index 15777eb8580..3b25134f5ff 100644 --- a/public/app/core/components/AppChrome/ReturnToPrevious/ReturnToPrevious.tsx +++ b/public/app/core/components/AppChrome/ReturnToPrevious/ReturnToPrevious.tsx @@ -45,6 +45,7 @@ const getStyles = (theme: GrafanaTheme2) => ({ zIndex: theme.zIndex.portal, position: 'fixed', bottom: theme.spacing.x4, + boxShadow: theme.shadows.z3, }), }); From 395a06ab86a67200733cb5718fc8d9a29239c04a Mon Sep 17 00:00:00 2001 From: Mihai Doarna Date: Wed, 31 Jan 2024 18:06:22 +0200 Subject: [PATCH 17/84] Auth: fix swagger responses for the SSO settings API (#81639) fix swagger responses for the sso settings API --- pkg/services/ssosettings/api/api.go | 20 ++++++++++-- public/api-merged.json | 48 +++++++++++++++++++++++++++-- public/openapi3.json | 48 +++++++++++++++++++++++++++-- 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/pkg/services/ssosettings/api/api.go b/pkg/services/ssosettings/api/api.go index 66357222650..4ebc206a137 100644 --- a/pkg/services/ssosettings/api/api.go +++ b/pkg/services/ssosettings/api/api.go @@ -233,7 +233,11 @@ type UpdateProviderSettingsParams struct { Provider string `json:"key"` // in:body // required:true - Body models.SSOSettings `json:"body"` + Body struct { + ID string `json:"id"` + Provider string `json:"provider"` + Settings map[string]any `json:"settings"` + } `json:"body"` } // swagger:parameters removeProviderSettings @@ -246,11 +250,21 @@ type RemoveProviderSettingsParams struct { // swagger:response listSSOSettingsResponse type ListSSOSettingsResponse struct { // in: body - Body []models.SSOSettings `json:"body"` + Body []struct { + ID string `json:"id"` + Provider string `json:"provider"` + Settings map[string]any `json:"settings"` + Source string `json:"source"` + } `json:"body"` } // swagger:response getSSOSettingsResponse type GetSSOSettingsResponse struct { // in: body - Body models.SSOSettings `json:"body"` + Body struct { + ID string `json:"id"` + Provider string `json:"provider"` + Settings map[string]any `json:"settings"` + Source string `json:"source"` + } `json:"body"` } diff --git a/public/api-merged.json b/public/api-merged.json index e30d02d361b..ee6218c9f04 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -11475,7 +11475,19 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/SSOSettings" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "type": "object", + "additionalProperties": false + } + } } } ], @@ -22991,7 +23003,22 @@ "getSSOSettingsResponse": { "description": "(empty)", "schema": { - "$ref": "#/definitions/SSOSettings" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "type": "object", + "additionalProperties": false + }, + "source": { + "type": "string" + } + } } }, "getSharingOptionsResponse": { @@ -23177,7 +23204,22 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/SSOSettings" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "type": "object", + "additionalProperties": false + }, + "source": { + "type": "string" + } + } } } }, diff --git a/public/openapi3.json b/public/openapi3.json index 2d6c2d08411..3410723e17e 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -1173,7 +1173,22 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SSOSettings" + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "additionalProperties": false, + "type": "object" + }, + "source": { + "type": "string" + } + }, + "type": "object" } } }, @@ -1438,7 +1453,22 @@ "application/json": { "schema": { "items": { - "$ref": "#/components/schemas/SSOSettings" + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "additionalProperties": false, + "type": "object" + }, + "source": { + "type": "string" + } + }, + "type": "object" }, "type": "array" } @@ -25205,7 +25235,19 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SSOSettings" + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "additionalProperties": false, + "type": "object" + } + }, + "type": "object" } } }, From cb945aa5df3453ab2a851ef7ec8bee244e3eeac4 Mon Sep 17 00:00:00 2001 From: Andrej Ocenas Date: Wed, 31 Jan 2024 17:26:12 +0100 Subject: [PATCH 18/84] NodeGraph: Use layered layout instead of force based layout (#78957) --- .drone.yml | 35 +- .yarnrc.yml | 4 + .../testdata/dataquery/schema-reference.md | 9 +- .../feature-toggles/index.md | 1 + package.json | 2 + .../src/types/featureToggles.gen.ts | 1 + .../x/TestDataDataQuery_types.gen.ts | 3 +- pkg/services/featuremgmt/registry.go | 8 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + .../kinds/dataquery/types_dataquery_gen.go | 8 +- .../app/features/plugins/built_in_plugins.ts | 4 +- .../datasource/azuremonitor/tsconfig.json | 2 +- .../components/NodeGraphEditor.tsx | 39 +- .../grafana-testdata-datasource/dataquery.cue | 3 +- .../dataquery.gen.ts | 3 +- .../grafana-testdata-datasource/datasource.ts | 11 +- .../nodeGraphUtils.ts | 23 +- .../grafana-testdata-datasource/package.json | 2 + .../testData/serviceMapResponse.ts | 27 +- .../testData/serviceMapResponseMedium.ts | 1004 +++++++++++++++++ .../panel/nodeGraph/createLayoutWorker.ts | 1 + public/app/plugins/panel/nodeGraph/layout.ts | 18 +- .../panel/nodeGraph/layoutMsagl.worker.js | 250 ++++ scripts/drone/steps/lib.star | 3 +- scripts/webpack/webpack.common.js | 7 + yarn.lock | 281 ++++- 27 files changed, 1684 insertions(+), 70 deletions(-) create mode 100644 public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponseMedium.ts create mode 100644 public/app/plugins/panel/nodeGraph/layoutMsagl.worker.js diff --git a/.drone.yml b/.drone.yml index 9434baae52d..47ff54c004d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -123,7 +123,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -225,7 +226,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -537,7 +539,8 @@ steps: image: golang:1.21.6-alpine name: wire-install - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1047,7 +1050,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1407,7 +1411,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1484,7 +1489,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1543,7 +1549,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1806,7 +1813,8 @@ steps: image: golang:1.21.6-alpine name: wire-install - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -2722,7 +2730,8 @@ steps: image: golang:1.21.6-alpine name: compile-build-cmd - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -2996,7 +3005,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -3420,7 +3430,8 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apt-get update -yq && apt-get install python -y + - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 + /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -4788,6 +4799,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: d61ada44fb898f089416a8f2afcea19481065e311a771ce4b08e03f2d13b4853 +hmac: 88526f5c0dd7e3e225914c39c331b827eb7846fd83be24d9e055c0aff72a7e67 ... diff --git a/.yarnrc.yml b/.yarnrc.yml index f891eb4e5c5..4f0f49b98d0 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -33,6 +33,10 @@ packageExtensions: react-resizable@3.0.4: peerDependencies: react-dom: 17.0.1 + '@msagl/drawing@*': + dependencies: + queue-typescript: "^1.0.1" + "@esfx/collections-sortedmap": "^1.0.0" plugins: - path: .yarn/plugins/@yarnpkg/plugin-outdated.cjs diff --git a/docs/sources/developers/kinds/composable/testdata/dataquery/schema-reference.md b/docs/sources/developers/kinds/composable/testdata/dataquery/schema-reference.md index 805542a6d3d..bc5da9a3aab 100644 --- a/docs/sources/developers/kinds/composable/testdata/dataquery/schema-reference.md +++ b/docs/sources/developers/kinds/composable/testdata/dataquery/schema-reference.md @@ -58,10 +58,11 @@ title: TestDataDataQuery kind ### NodesQuery -| Property | Type | Required | Default | Description | -|----------|---------|----------|---------|------------------------------------------------------------| -| `count` | integer | No | | | -| `type` | string | No | | Possible values are: `random`, `response`, `random edges`. | +| Property | Type | Required | Default | Description | +|----------|---------|----------|---------|-------------------------------------------------------------------------------------| +| `count` | integer | No | | | +| `seed` | integer | No | | | +| `type` | string | No | | Possible values are: `random`, `response_small`, `response_medium`, `random edges`. | ### PulseWaveQuery diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 9b785a3d3f0..b92749e4304 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -174,6 +174,7 @@ Experimental features might be changed or removed without prior notice. | `newFolderPicker` | Enables the nested folder picker without having nested folders enabled | | `onPremToCloudMigrations` | In-development feature that will allow users to easily migrate their on-prem Grafana instances to Grafana Cloud. | | `promQLScope` | In-development feature that will allow injection of labels into prometheus queries. | +| `nodeGraphDotLayout` | Changed the layout algorithm for the node graph | ## Development feature toggles diff --git a/package.json b/package.json index 9d797b77b77..f156e1c9730 100644 --- a/package.json +++ b/package.json @@ -257,6 +257,8 @@ "@locker/near-membrane-dom": "0.13.3", "@locker/near-membrane-shared": "0.13.3", "@locker/near-membrane-shared-dom": "0.13.3", + "@msagl/core": "^1.1.16", + "@msagl/parser": "^1.1.16", "@opentelemetry/api": "1.7.0", "@opentelemetry/exporter-collector": "0.25.0", "@opentelemetry/semantic-conventions": "1.21.0", diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 8ffcae0e84e..75f54ac9821 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -175,4 +175,5 @@ export interface FeatureToggles { alertingSaveStatePeriodic?: boolean; promQLScope?: boolean; slateAutocomplete?: boolean; + nodeGraphDotLayout?: boolean; } diff --git a/packages/grafana-schema/src/raw/composable/testdata/dataquery/x/TestDataDataQuery_types.gen.ts b/packages/grafana-schema/src/raw/composable/testdata/dataquery/x/TestDataDataQuery_types.gen.ts index 01233d78f7c..91f7cbe3bdb 100644 --- a/packages/grafana-schema/src/raw/composable/testdata/dataquery/x/TestDataDataQuery_types.gen.ts +++ b/packages/grafana-schema/src/raw/composable/testdata/dataquery/x/TestDataDataQuery_types.gen.ts @@ -75,7 +75,8 @@ export interface SimulationQuery { export interface NodesQuery { count?: number; - type?: ('random' | 'response' | 'random edges'); + seed?: number; + type?: ('random' | 'response_small' | 'response_medium' | 'random edges'); } export interface USAQuery { diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 44d5b325147..bfab3b7a3eb 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1329,5 +1329,13 @@ var ( Owner: grafanaFrontendPlatformSquad, Created: time.Date(2024, time.January, 29, 12, 0, 0, 0, time.UTC), }, + { + Name: "nodeGraphDotLayout", + Description: "Changed the layout algorithm for the node graph", + Stage: FeatureStageExperimental, + FrontendOnly: true, + Owner: grafanaObservabilityTracesAndProfilingSquad, + Created: time.Date(2024, time.January, 2, 12, 0, 0, 0, time.UTC), + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 18ac3654d23..197563171e8 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -156,3 +156,4 @@ onPremToCloudMigrations,experimental,@grafana/grafana-operator-experience-squad, alertingSaveStatePeriodic,privatePreview,@grafana/alerting-squad,2024-01-22,false,false,false promQLScope,experimental,@grafana/observability-metrics,2024-01-29,false,false,false slateAutocomplete,GA,@grafana/grafana-frontend-platform,2024-01-29,false,false,true +nodeGraphDotLayout,experimental,@grafana/observability-traces-and-profiling,2024-01-02,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 150ec059934..2c0e677c0c3 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -634,4 +634,8 @@ const ( // FlagSlateAutocomplete // Adjusts the behaviour of the slate editor to properly handle autocomplete. Feature toggled for safety. FlagSlateAutocomplete = "slateAutocomplete" + + // FlagNodeGraphDotLayout + // Changed the layout algorithm for the node graph + FlagNodeGraphDotLayout = "nodeGraphDotLayout" ) diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/dataquery/types_dataquery_gen.go b/pkg/tsdb/grafana-testdata-datasource/kinds/dataquery/types_dataquery_gen.go index 2475dff45e8..3013282e006 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/dataquery/types_dataquery_gen.go +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/dataquery/types_dataquery_gen.go @@ -11,9 +11,10 @@ package dataquery // Defines values for NodesQueryType. const ( - NodesQueryTypeRandom NodesQueryType = "random" - NodesQueryTypeRandomEdges NodesQueryType = "random edges" - NodesQueryTypeResponse NodesQueryType = "response" + NodesQueryTypeRandom NodesQueryType = "random" + NodesQueryTypeRandomEdges NodesQueryType = "random edges" + NodesQueryTypeResponseMedium NodesQueryType = "response_medium" + NodesQueryTypeResponseSmall NodesQueryType = "response_small" ) // Defines values for StreamingQueryType. @@ -100,6 +101,7 @@ type DataQuery struct { // NodesQuery defines model for NodesQuery. type NodesQuery struct { Count *int64 `json:"count,omitempty"` + Seed *int64 `json:"seed,omitempty"` Type *NodesQueryType `json:"type,omitempty"` } diff --git a/public/app/features/plugins/built_in_plugins.ts b/public/app/features/plugins/built_in_plugins.ts index 419f6e10a63..0287e6686de 100644 --- a/public/app/features/plugins/built_in_plugins.ts +++ b/public/app/features/plugins/built_in_plugins.ts @@ -58,7 +58,6 @@ import * as histogramPanel from 'app/plugins/panel/histogram/module'; import * as livePanel from 'app/plugins/panel/live/module'; import * as logsPanel from 'app/plugins/panel/logs/module'; import * as newsPanel from 'app/plugins/panel/news/module'; -import * as nodeGraph from 'app/plugins/panel/nodeGraph/module'; import * as pieChartPanel from 'app/plugins/panel/piechart/module'; import * as statPanel from 'app/plugins/panel/stat/module'; import * as stateTimelinePanel from 'app/plugins/panel/state-timeline/module'; @@ -80,6 +79,9 @@ const heatmapPanel = async () => const tableOldPanel = async () => await import(/* webpackChunkName: "tableOldPlugin" */ 'app/plugins/panel/table-old/module'); +const nodeGraph = async () => + await import(/* webpackChunkName: "nodeGraphPanel" */ 'app/plugins/panel/nodeGraph/module'); + const builtInPlugins: Record Promise)> = { // datasources 'core:plugin/graphite': graphitePlugin, diff --git a/public/app/plugins/datasource/azuremonitor/tsconfig.json b/public/app/plugins/datasource/azuremonitor/tsconfig.json index 7daf2ee8aba..9ace7d78dbf 100644 --- a/public/app/plugins/datasource/azuremonitor/tsconfig.json +++ b/public/app/plugins/datasource/azuremonitor/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "@grafana/plugin-configs/tsconfig.json", - "include": ["."] + "include": ["."], } diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/components/NodeGraphEditor.tsx b/public/app/plugins/datasource/grafana-testdata-datasource/components/NodeGraphEditor.tsx index 1c804b569a1..4421fe6627e 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/components/NodeGraphEditor.tsx +++ b/public/app/plugins/datasource/grafana-testdata-datasource/components/NodeGraphEditor.tsx @@ -24,21 +24,34 @@ export function NodeGraphEditor({ query, onChange }: Props) { /> {(type === 'random' || type === 'random edges') && ( - - - onChange({ ...query.nodes, count: e.currentTarget.value ? parseInt(e.currentTarget.value, 10) : 0 }) - } - placeholder="10" - /> - + <> + + + onChange({ ...query.nodes, count: e.currentTarget.value ? parseInt(e.currentTarget.value, 10) : 0 }) + } + placeholder="10" + /> + + + + onChange({ ...query.nodes, seed: e.currentTarget.value ? parseInt(e.currentTarget.value, 10) : 0 }) + } + /> + + )} ); } -const options: Array = ['random', 'response', 'random edges']; +const options: Array = ['random', 'response_small', 'response_medium', 'random edges']; diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.cue b/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.cue index a250f4f3dca..da51680346b 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.cue +++ b/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.cue @@ -83,8 +83,9 @@ composableKinds: DataQuery: { } @cuetsy(kind="interface") #NodesQuery: { - type?: "random" | "response" | "random edges" + type?: "random" | "response_small" | "response_medium" | "random edges" count?: int64 + seed?: int64 } @cuetsy(kind="interface") #USAQuery: { diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.gen.ts b/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.gen.ts index 6051914bc6f..ed487a37c29 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.gen.ts +++ b/public/app/plugins/datasource/grafana-testdata-datasource/dataquery.gen.ts @@ -72,7 +72,8 @@ export interface SimulationQuery { export interface NodesQuery { count?: number; - type?: ('random' | 'response' | 'random edges'); + seed?: number; + type?: ('random' | 'response_small' | 'response_medium' | 'random edges'); } export interface USAQuery { diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/datasource.ts b/public/app/plugins/datasource/grafana-testdata-datasource/datasource.ts index 795c7388266..e290b34a022 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/datasource.ts +++ b/public/app/plugins/datasource/grafana-testdata-datasource/datasource.ts @@ -232,13 +232,16 @@ export class TestDataDataSource extends DataSourceWithBackend { let frames: DataFrame[]; switch (type) { case 'random': - frames = generateRandomNodes(target.nodes?.count); + frames = generateRandomNodes(target.nodes?.count, target.nodes?.seed); break; - case 'response': - frames = savedNodesResponse(); + case 'response_small': + frames = savedNodesResponse('small'); + break; + case 'response_medium': + frames = savedNodesResponse('medium'); break; case 'random edges': - frames = [generateRandomEdges(target.nodes?.count)]; + frames = [generateRandomEdges(target.nodes?.count, target.nodes?.seed)]; break; default: throw new Error(`Unknown node_graph sub type ${type}`); diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/nodeGraphUtils.ts b/public/app/plugins/datasource/grafana-testdata-datasource/nodeGraphUtils.ts index 515d3c885c2..c575d25b8e7 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/nodeGraphUtils.ts +++ b/public/app/plugins/datasource/grafana-testdata-datasource/nodeGraphUtils.ts @@ -1,3 +1,5 @@ +import { randomLcg } from 'd3-random'; + import { FieldColorModeId, FieldDTO, @@ -7,11 +9,13 @@ import { DataFrame, } from '@grafana/data'; -import { nodes, edges } from './testData/serviceMapResponse'; +import * as serviceMapResponseSmall from './testData/serviceMapResponse'; +import * as serviceMapResponsMedium from './testData/serviceMapResponseMedium'; -export function generateRandomNodes(count = 10) { +export function generateRandomNodes(count = 10, seed?: number) { const nodes = []; const edges: string[] = []; + const rand = randomLcg(seed); const root = { id: 'root', @@ -31,7 +35,7 @@ export function generateRandomNodes(count = 10) { for (let i = 1; i < count; i++) { const node = makeRandomNode(i); nodes.push(node); - const sourceIndex = Math.floor(Math.random() * Math.floor(nodesWithoutMaxEdges.length - 1)); + const sourceIndex = Math.floor(rand() * Math.floor(nodesWithoutMaxEdges.length - 1)); const source = nodesWithoutMaxEdges[sourceIndex]; source.edges.push(node.id); if (source.edges.length >= maxEdges) { @@ -43,8 +47,8 @@ export function generateRandomNodes(count = 10) { // Add some random edges to create possible cycle const additionalEdges = Math.floor(count / 2); for (let i = 0; i <= additionalEdges; i++) { - const sourceIndex = Math.floor(Math.random() * Math.floor(nodes.length - 1)); - const targetIndex = Math.floor(Math.random() * Math.floor(nodes.length - 1)); + const sourceIndex = Math.floor(rand() * Math.floor(nodes.length - 1)); + const targetIndex = Math.floor(rand() * Math.floor(nodes.length - 1)); if (sourceIndex === targetIndex || nodes[sourceIndex].id === '0' || nodes[targetIndex].id === '0') { continue; } @@ -185,11 +189,12 @@ function makeRandomNode(index: number) { }; } -export function savedNodesResponse() { - return [new MutableDataFrame(nodes), new MutableDataFrame(edges)]; +export function savedNodesResponse(size: 'small' | 'medium'): [DataFrame, DataFrame] { + const response = size === 'small' ? serviceMapResponseSmall : serviceMapResponsMedium; + return [new MutableDataFrame(response.nodes), new MutableDataFrame(response.edges)]; } // Generates node graph data but only returns the edges -export function generateRandomEdges(count = 10) { - return generateRandomNodes(count)[1]; +export function generateRandomEdges(count = 10, seed = 1) { + return generateRandomNodes(count, seed)[1]; } diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/package.json b/public/app/plugins/datasource/grafana-testdata-datasource/package.json index 95fb7b28974..547ed50b2db 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/package.json +++ b/public/app/plugins/datasource/grafana-testdata-datasource/package.json @@ -10,6 +10,7 @@ "@grafana/runtime": "10.4.0-pre", "@grafana/schema": "10.4.0-pre", "@grafana/ui": "10.4.0-pre", + "d3-random": "^3.0.1", "lodash": "4.17.21", "micro-memoize": "^4.1.2", "react": "18.2.0", @@ -23,6 +24,7 @@ "@grafana/plugin-configs": "10.4.0-pre", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", + "@types/d3-random": "^3.0.2", "@types/jest": "29.5.11", "@types/lodash": "4.14.202", "@types/node": "20.11.13", diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponse.ts b/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponse.ts index dabecebcfc4..bd0ea583d96 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponse.ts +++ b/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponse.ts @@ -57,7 +57,7 @@ export const nodes = { }, ], }, - values: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + values: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], }, { name: NodeGraphDataFrameFieldNames.title, @@ -79,6 +79,8 @@ export const nodes = { 'api', 'www', 'products', + 'test-client', + 'test-api', ], }, { @@ -101,6 +103,8 @@ export const nodes = { 'client', 'client', 'Compute', + 'ephemeral', + 'ephemeral-api', ], }, { @@ -110,7 +114,7 @@ export const nodes = { values: [ 3.5394042646735553, 15.906441318223264, 4.913011921591567, 7.4163203042094095, 1092, 22.85961441405067, 56.135855729084696, 4.45946191601527, 12.818300278280843, 4.25, 12.565442646791492, 77.63447512700567, - 40.387096774193544, 77.63447512700567, 27.648950187374872, + 40.387096774193544, 77.63447512700567, 27.648950187374872, 77.63447512700567, 27.648950187374872, ], }, { @@ -120,7 +124,7 @@ export const nodes = { values: [ 50.56317154501667, 682.4, 512.8416666666667, 125.64444444444445, 0.005585812037424941, 137.59722222222223, 300.0527777777778, 30.582348853370394, 125.77222222222223, 0.028706417080318163, 30.582348853370394, 165.675, - 0.100021510002151, 165.675, 162.33055555555555, + 0.100021510002151, 165.675, 162.33055555555555, 165.675, 162.33055555555555, ], }, { @@ -129,7 +133,7 @@ export const nodes = { config: { color: { mode: FieldColorModeId.Fixed, fixedColor: 'green' }, displayName: 'Sucesss' }, values: [ 0.9338865684765882, 1, 1, 1, 0.5, 1, 0.9901128505170387, 0.9069260134520997, 1, 0, 0.9069260134520997, - 0.9624432037288534, 0, 0.9624432037288534, 0.9824945669843769, + 0.9624432037288534, 0, 0.9624432037288534, 0.9824945669843769, 0.9624432037288534, 0.9824945669843769, ], }, { @@ -138,7 +142,7 @@ export const nodes = { config: { color: { mode: FieldColorModeId.Fixed, fixedColor: 'red' }, displayName: 'Faults' }, values: [ 0, 0, 0, 0, 0.5, 0, 0.009479813736472288, 0, 0, 0, 0, 0.017168821152524185, 0, 0.017168821152524185, - 0.01750543301562313, + 0.01750543301562313, 0.017168821152524185, 0.01750543301562313, ], }, { @@ -147,14 +151,14 @@ export const nodes = { config: { color: { mode: FieldColorModeId.Fixed, fixedColor: 'semi-dark-yellow' }, displayName: 'Errors' }, values: [ 0.06611343152341174, 0, 0, 0, 0, 0, 0.0004073357464890436, 0.09307398654790038, 0, 1, 0.09307398654790038, - 0.02038797511862247, 1, 0.02038797511862247, 0, + 0.02038797511862247, 1, 0.02038797511862247, 0, 0.02038797511862247, 0, ], }, { name: NodeGraphDataFrameFieldNames.arc + 'throttled', type: FieldType.number, config: { color: { mode: FieldColorModeId.Fixed, fixedColor: 'purple' }, displayName: 'Throttled' }, - values: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + values: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], }, ], meta: { preferredVisualisationType: 'nodeGraph' as const }, @@ -236,13 +240,14 @@ export const edges = { '14__1', '14__2', '14__10', + '15__16', ], }, { name: NodeGraphDataFrameFieldNames.source, type: FieldType.string, config: {}, - values: [0, 5, 6, 6, 6, 6, 6, 6, 8, 10, 11, 11, 12, 13, 14, 14, 14], + values: [0, 5, 6, 6, 6, 6, 6, 6, 8, 10, 11, 11, 12, 13, 14, 14, 14, 15], }, { name: 'sourceName', @@ -266,13 +271,14 @@ export const edges = { 'products', 'products', 'products', + 'test-client', ], }, { name: NodeGraphDataFrameFieldNames.target, type: FieldType.string, config: {}, - values: [2, 8, 0, 5, 9, 2, 14, 4, 3, 7, 0, 6, 6, 11, 1, 2, 10], + values: [2, 8, 0, 5, 9, 2, 14, 4, 3, 7, 0, 6, 6, 11, 1, 2, 10, 16], }, { name: 'targetName', @@ -296,6 +302,7 @@ export const edges = { 'products', 'customers', 'shipping', + 'test-api', ], }, { @@ -320,6 +327,7 @@ export const edges = { 'Success 100.00%', 'Success 100.00%', 'Faults 9.30%', + 'Faults 9.30%', ], }, { @@ -330,6 +338,7 @@ export const edges = { 50.56317154501667, 125.77222222222223, 0.03333333333333333, 137.59722222222223, 0.022222222222222223, 299.96666666666664, 162.33055555555555, 0.005555555555555556, 125.64444444444445, 30.582348853370394, 50.51111111111111, 299.9166666666667, 0.100021510002151, 165.675, 682.4, 162.33055555555555, 30.558333333333334, + 30.558333333333334, ], }, ], diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponseMedium.ts b/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponseMedium.ts new file mode 100644 index 00000000000..ac0b2599c08 --- /dev/null +++ b/public/app/plugins/datasource/grafana-testdata-datasource/testData/serviceMapResponseMedium.ts @@ -0,0 +1,1004 @@ +import { FieldType, NodeGraphDataFrameFieldNames } from '@grafana/data'; + +export const nodes = { + fields: [ + { + name: NodeGraphDataFrameFieldNames.id, + type: FieldType.string, + values: [ + '0', + 'candycon2000', + 'pantry-summary', + 'compute-splines-api', + 'send-notification', + 'popcorn-distributor', + 'caramel-gateway', + 'candy-distributor', + 'candy-query-frontend', + 'candy-ruler', + 'candy-querier', + 'candy-chocolate-querier', + 'candy-backend', + 'candy-read', + 'candy-write', + 'popcorn-backend', + 'popcorn-query-frontend', + 'proxy-reads', + 'proxy-gateway', + 'proxy-writes', + 'dentist', + 'fudge-proxy-gateway', + 'candy-admin-api', + 'candy-disposer', + 'candy-ingester', + 'candy-gateway', + 'candy-chocolate-write-proxy', + 'candy-store-gateway', + 'popcorn-querier', + 'candy-query-scheduler', + 'api', + 'controller', + 'gateway', + 'db', + 'marshmallow-popcorn-plugin', + 'eggnog-query', + 'licorice-ingester', + 'licorice-distributor', + 'licorice-gateway', + 'licorice-querier', + 'licorice-query-frontend', + 'licorice-ruler', + 'licorice-read', + 'licorice-write', + 'licorice-sugar-gateway', + 'licorice-query-scheduler', + 'popcorn-ingester', + 'popcorn-ruler', + 'peanutapi', + 'almondapi', + 'worker', + 'toffee-gateway', + 'marshmallow-gateway', + 'toffee-proxy-gateway', + 'mints-distributor', + 'mints-agent', + 'mints-ingester', + 'marshmallow-query-frontend', + 'marshmallow-ingester', + 'marshmallow-querier', + 'eating-scheduler', + 'calorie-generator', + 'candy-packaging-exporter', + 'candy-manager', + 'overeat-gateway', + 'wrappers-api', + 'overeat_api', + 'licorice-admin-api', + 'almond', + 'almondops-gateway', + ], + }, + { + name: NodeGraphDataFrameFieldNames.title, + config: { + displayName: 'Service name', + }, + type: FieldType.string, + values: [ + '0', + 'candycon2000', + 'pantry-summary', + 'compute-splines-api', + 'send-notification', + 'popcorn-distributor', + 'caramel-gateway', + 'candy-distributor', + 'candy-query-frontend', + 'candy-ruler', + 'candy-querier', + 'candy-chocolate-querier', + 'candy-backend', + 'candy-read', + 'candy-write', + 'popcorn-backend', + 'popcorn-query-frontend', + 'proxy-reads', + 'proxy-gateway', + 'proxy-writes', + 'dentist', + 'fudge-proxy-gateway', + 'candy-admin-api', + 'candy-disposer', + 'candy-ingester', + 'candy-gateway', + 'candy-chocolate-write-proxy', + 'candy-store-gateway', + 'popcorn-querier', + 'candy-query-scheduler', + 'api', + 'controller', + 'gateway', + 'db', + 'marshmallow-popcorn-plugin', + 'eggnog-query', + 'licorice-ingester', + 'licorice-distributor', + 'licorice-gateway', + 'licorice-querier', + 'licorice-query-frontend', + 'licorice-ruler', + 'licorice-read', + 'licorice-write', + 'licorice-sugar-gateway', + 'licorice-query-scheduler', + 'popcorn-ingester', + 'popcorn-ruler', + 'peanutapi', + 'almondapi', + 'worker', + 'toffee-gateway', + 'marshmallow-gateway', + 'toffee-proxy-gateway', + 'mints-distributor', + 'mints-agent', + 'mints-ingester', + 'marshmallow-query-frontend', + 'marshmallow-ingester', + 'marshmallow-querier', + 'eating-scheduler', + 'calorie-generator', + 'candy-packaging-exporter', + 'candy-manager', + 'overeat-gateway', + 'wrappers-api', + 'overeat_api', + 'licorice-admin-api', + 'almond', + 'almondops-gateway', + ], + }, + { + name: NodeGraphDataFrameFieldNames.mainStat, + config: { + unit: 'ms/r', + displayName: 'Average response time', + }, + type: FieldType.number, + values: [ + 0.3792332384891563, + 0.8672229016348518, + null, + 16.20695699632165, + 39.928285334583585, + 9.655124202033475, + 21.05268555627574, + 5.867123115073844, + 2.4860418856893776, + 355.53214166441546, + 20.55178572103791, + null, + 38.89817349596472, + 8.180751216025271, + 0.8054812870678301, + 40.77678203483556, + 85.482797395998, + 16.27026063145621, + null, + 19.87661445917064, + 0.22942839760131845, + null, + null, + null, + 0.5613453804178244, + null, + null, + 8.992145179555344, + 304.9330783211414, + null, + 10.395531440216706, + null, + null, + null, + 0.015141602590812384, + null, + 2516.1181289425012, + 1.753552009081408, + 2.18723509171844, + 2042.6033859578174, + 35.584947433232884, + 12.650689593313736, + 11469.63275426663, + 10032.233577903402, + 2.7579124143530875, + null, + 12.313124260662056, + null, + 449.2619110960284, + null, + 457.68439141665425, + 4.200223281332062, + 3.22412985358373, + null, + 16.03316977154454, + null, + 5.830408312271875, + 1460.8431543525896, + 6.379590630002257, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + ], + }, + { + name: NodeGraphDataFrameFieldNames.secondaryStat, + config: { + unit: 'r/sec', + displayName: 'Requests per second', + }, + type: FieldType.number, + values: [ + 0, + 0, + null, + 0, + 0, + 0, + 0, + 0, + 0.01, + 0, + 0.01, + null, + 0, + 0, + 0.02, + 0, + 0, + 0, + null, + 0, + 0, + null, + null, + null, + 0.1, + null, + null, + 0, + 0, + null, + 0, + null, + null, + null, + 0, + null, + 0.06, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + 0, + null, + 0, + null, + 0, + 0, + 0, + null, + 0, + null, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + ], + }, + { + name: 'arc__success', + config: { + displayName: 'Success', + color: { + fixedColor: 'green', + mode: 'fixed', + }, + }, + type: FieldType.number, + values: [ + 1, 1, 1, 1, 0, 1, 0.999931271795046, 0.9999982854540072, 0.9999993106537873, 1, 0.9999988795927239, 1, + 0.9984939706365067, 0.9999936988458762, 0.9374731810143394, 0.9981691514900277, 1, 1, 1, 0.6918050349578433, 1, + 1, 1, 1, 0.9649904403274051, 1, 1, 0.996693264215713, 0.9990646675327494, 1, 0.989711501101388, 1, 1, 1, 1, 1, + 0.974376082211577, 1, 1, 0.9979501689527777, 0.9999554801847671, 0.7997517799066115, 0.998393334495106, + 0.8077280841753209, 0.9999889402832068, 1, 0.9267936098112096, 1, 0.9678190225122817, 1, 0.9664342727367723, 1, + 1, 1, 1, 1, 1, 0.49879813934581896, 0.8514429797914538, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ], + }, + { + name: 'arc__failed', + config: { + displayName: 'Failed', + color: { + fixedColor: 'red', + mode: 'fixed', + }, + }, + type: FieldType.number, + values: [ + 0, 0, 0, 0, 1, 0, 0.00006872820495400671, 0.0000017145459928594954, 6.893462126672767e-7, 0, + 0.000001120407275990985, 0, 0.001506029363493422, 0.000006301154123861364, 0.06252681898566055, + 0.0018308485099723893, 0, 0, 0, 0.30819496504215677, 0, 0, 0, 0, 0.03500955967259495, 0, 0, + 0.0033067357842869243, 0.0009353324672505954, 0, 0.01028849889861211, 0, 0, 0, 0, 0, 0.025623917788423094, 0, 0, + 0.002049831047222383, 0.00004451981523281711, 0.20024822009338844, 0.001606665504893849, 0.19227191582467903, + 0.000011059716793143872, 0, 0.07320639018879034, 0, 0.032180977487718344, 0, 0.03356572726322771, 0, 0, 0, 0, 0, + 0, 0.501201860654181, 0.14855702020854625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ], + }, + ], + meta: { preferredVisualisationType: 'nodeGraph' as const }, + name: 'nodes', +}; + +export const edges = { + fields: [ + { + name: NodeGraphDataFrameFieldNames.id, + type: FieldType.string, + values: [ + 'pantry-summary_candycon2000', + 'compute-splines-api_candycon2000', + 'compute-splines-api_send-notification', + 'caramel-gateway_popcorn-distributor', + 'caramel-gateway_candy-distributor', + 'caramel-gateway_candy-query-frontend', + 'caramel-gateway_candy-ruler', + 'caramel-gateway_caramel-gateway', + 'caramel-gateway_candy-querier', + 'caramel-gateway_candy-chocolate-querier', + 'caramel-gateway_candy-backend', + 'caramel-gateway_candy-read', + 'caramel-gateway_candy-write', + 'caramel-gateway_popcorn-backend', + 'caramel-gateway_popcorn-query-frontend', + 'proxy-gateway_proxy-reads', + 'proxy-gateway_proxy-writes', + 'proxy-reads_caramel-gateway', + 'proxy-writes_caramel-gateway', + 'fudge-proxy-gateway_dentist', + 'candy-admin-api_candy-distributor', + 'candy-backend_candy-read', + 'candy-backend_candy-write', + 'candy-disposer_candy-distributor', + 'candy-distributor_candy-ingester', + 'candy-distributor_candy-distributor', + 'candy-gateway_candy-admin-api', + 'candy-gateway_candy-distributor', + 'candy-gateway_candy-query-frontend', + 'candy-chocolate-write-proxy_caramel-gateway', + 'candy-ingester_candy-distributor', + 'candy-querier_candy-ingester', + 'candy-querier_candy-query-frontend', + 'candy-querier_candy-store-gateway', + 'candy-querier_candy-distributor', + 'candy-query-frontend_candy-querier', + 'candy-query-frontend_popcorn-querier', + 'candy-query-frontend_candy-distributor', + 'candy-query-scheduler_candy-distributor', + 'candy-read_candy-backend', + 'candy-read_candy-read', + 'candy-read_candy-write', + 'candy-read_popcorn-backend', + 'candy-ruler_candy-ruler', + 'candy-ruler_candy-ingester', + 'candy-ruler_candy-query-frontend', + 'candy-ruler_candy-distributor', + 'candy-write_candy-write', + 'controller_api', + 'gateway_api', + 'db_api', + 'eggnog-query_marshmallow-popcorn-plugin', + 'licorice-distributor_licorice-ingester', + 'licorice-gateway_licorice-distributor', + 'licorice-gateway_licorice-querier', + 'licorice-gateway_licorice-query-frontend', + 'licorice-gateway_licorice-ruler', + 'licorice-gateway_licorice-read', + 'licorice-gateway_licorice-write', + 'licorice-querier_licorice-sugar-gateway', + 'licorice-querier_licorice-ingester', + 'licorice-querier_licorice-query-frontend', + 'licorice-query-frontend_licorice-querier', + 'licorice-query-scheduler_licorice-query-frontend', + 'licorice-read_licorice-read', + 'licorice-read_licorice-write', + 'licorice-ruler_licorice-ingester', + 'licorice-ruler_licorice-sugar-gateway', + 'licorice-ruler_licorice-ruler', + 'licorice-write_licorice-write', + 'popcorn-backend_candy-read', + 'popcorn-backend_candy-write', + 'popcorn-distributor_candy-ingester', + 'popcorn-distributor_popcorn-ingester', + 'popcorn-querier_candy-ingester', + 'popcorn-querier_candy-query-frontend', + 'popcorn-querier_candy-store-gateway', + 'popcorn-querier_popcorn-ingester', + 'popcorn-querier_popcorn-query-frontend', + 'popcorn-query-frontend_popcorn-querier', + 'popcorn-ruler_candy-ingester', + 'popcorn-ruler_popcorn-query-frontend', + 'almondapi_peanutapi', + 'peanutapi_0', + 'peanutapi_worker', + 'toffee-gateway_licorice-gateway', + 'toffee-gateway_marshmallow-gateway', + 'toffee-proxy-gateway_toffee-gateway', + 'mints-agent_mints-distributor', + 'mints-distributor_mints-ingester', + 'marshmallow-gateway_marshmallow-query-frontend', + 'marshmallow-querier_marshmallow-ingester', + 'eating-scheduler_peanutapi', + 'calorie-generator_compute-splines-api', + 'worker_0', + 'worker_worker', + 'candy-backend_candy-backend', + 'candy-gateway_candy-ruler', + 'candy-packaging-exporter_candy-distributor', + 'caramel-gateway_candy-chocolate-write-proxy', + 'candy-manager_candy-distributor', + 'candy-backend_popcorn-backend', + 'candy-gateway_candy-manager', + 'candy-gateway_candy-querier', + 'candy-gateway_candy-store-gateway', + 'candy-gateway_candy-disposer', + 'candy-query-scheduler_candy-query-frontend', + 'candy-store-gateway_candy-distributor', + 'api_gateway', + 'gateway_overeat-gateway', + 'gateway_wrappers-api', + 'overeat-gateway_overeat_api', + 'overeat_api_gateway', + 'wrappers-api_caramel-gateway', + 'licorice-gateway_licorice-admin-api', + 'popcorn-backend_popcorn-backend', + 'popcorn-backend_candy-backend', + 'almondops-gateway_almond', + 'marshmallow-gateway_marshmallow-querier', + ], + }, + { + name: NodeGraphDataFrameFieldNames.source, + type: FieldType.string, + config: {}, + values: [ + 'pantry-summary', + 'compute-splines-api', + 'compute-splines-api', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'caramel-gateway', + 'proxy-gateway', + 'proxy-gateway', + 'proxy-reads', + 'proxy-writes', + 'fudge-proxy-gateway', + 'candy-admin-api', + 'candy-backend', + 'candy-backend', + 'candy-disposer', + 'candy-distributor', + 'candy-distributor', + 'candy-gateway', + 'candy-gateway', + 'candy-gateway', + 'candy-chocolate-write-proxy', + 'candy-ingester', + 'candy-querier', + 'candy-querier', + 'candy-querier', + 'candy-querier', + 'candy-query-frontend', + 'candy-query-frontend', + 'candy-query-frontend', + 'candy-query-scheduler', + 'candy-read', + 'candy-read', + 'candy-read', + 'candy-read', + 'candy-ruler', + 'candy-ruler', + 'candy-ruler', + 'candy-ruler', + 'candy-write', + 'controller', + 'gateway', + 'db', + 'eggnog-query', + 'licorice-distributor', + 'licorice-gateway', + 'licorice-gateway', + 'licorice-gateway', + 'licorice-gateway', + 'licorice-gateway', + 'licorice-gateway', + 'licorice-querier', + 'licorice-querier', + 'licorice-querier', + 'licorice-query-frontend', + 'licorice-query-scheduler', + 'licorice-read', + 'licorice-read', + 'licorice-ruler', + 'licorice-ruler', + 'licorice-ruler', + 'licorice-write', + 'popcorn-backend', + 'popcorn-backend', + 'popcorn-distributor', + 'popcorn-distributor', + 'popcorn-querier', + 'popcorn-querier', + 'popcorn-querier', + 'popcorn-querier', + 'popcorn-querier', + 'popcorn-query-frontend', + 'popcorn-ruler', + 'popcorn-ruler', + 'almondapi', + 'peanutapi', + 'peanutapi', + 'toffee-gateway', + 'toffee-gateway', + 'toffee-proxy-gateway', + 'mints-agent', + 'mints-distributor', + 'marshmallow-gateway', + 'marshmallow-querier', + 'eating-scheduler', + 'calorie-generator', + 'worker', + 'worker', + 'candy-backend', + 'candy-gateway', + 'candy-packaging-exporter', + 'caramel-gateway', + 'candy-manager', + 'candy-backend', + 'candy-gateway', + 'candy-gateway', + 'candy-gateway', + 'candy-gateway', + 'candy-query-scheduler', + 'candy-store-gateway', + 'api', + 'gateway', + 'gateway', + 'overeat-gateway', + 'overeat_api', + 'wrappers-api', + 'licorice-gateway', + 'popcorn-backend', + 'popcorn-backend', + 'almondops-gateway', + 'marshmallow-gateway', + ], + }, + { + name: NodeGraphDataFrameFieldNames.target, + type: FieldType.string, + config: {}, + values: [ + 'candycon2000', + 'candycon2000', + 'send-notification', + 'popcorn-distributor', + 'candy-distributor', + 'candy-query-frontend', + 'candy-ruler', + 'caramel-gateway', + 'candy-querier', + 'candy-chocolate-querier', + 'candy-backend', + 'candy-read', + 'candy-write', + 'popcorn-backend', + 'popcorn-query-frontend', + 'proxy-reads', + 'proxy-writes', + 'caramel-gateway', + 'caramel-gateway', + 'dentist', + 'candy-distributor', + 'candy-read', + 'candy-write', + 'candy-distributor', + 'candy-ingester', + 'candy-distributor', + 'candy-admin-api', + 'candy-distributor', + 'candy-query-frontend', + 'caramel-gateway', + 'candy-distributor', + 'candy-ingester', + 'candy-query-frontend', + 'candy-store-gateway', + 'candy-distributor', + 'candy-querier', + 'popcorn-querier', + 'candy-distributor', + 'candy-distributor', + 'candy-backend', + 'candy-read', + 'candy-write', + 'popcorn-backend', + 'candy-ruler', + 'candy-ingester', + 'candy-query-frontend', + 'candy-distributor', + 'candy-write', + 'api', + 'api', + 'api', + 'marshmallow-popcorn-plugin', + 'licorice-ingester', + 'licorice-distributor', + 'licorice-querier', + 'licorice-query-frontend', + 'licorice-ruler', + 'licorice-read', + 'licorice-write', + 'licorice-sugar-gateway', + 'licorice-ingester', + 'licorice-query-frontend', + 'licorice-querier', + 'licorice-query-frontend', + 'licorice-read', + 'licorice-write', + 'licorice-ingester', + 'licorice-sugar-gateway', + 'licorice-ruler', + 'licorice-write', + 'candy-read', + 'candy-write', + 'candy-ingester', + 'popcorn-ingester', + 'candy-ingester', + 'candy-query-frontend', + 'candy-store-gateway', + 'popcorn-ingester', + 'popcorn-query-frontend', + 'popcorn-querier', + 'candy-ingester', + 'popcorn-query-frontend', + 'peanutapi', + '0', + 'worker', + 'licorice-gateway', + 'marshmallow-gateway', + 'toffee-gateway', + 'mints-distributor', + 'mints-ingester', + 'marshmallow-query-frontend', + 'marshmallow-ingester', + 'peanutapi', + 'compute-splines-api', + '0', + 'worker', + 'candy-backend', + 'candy-ruler', + 'candy-distributor', + 'candy-chocolate-write-proxy', + 'candy-distributor', + 'popcorn-backend', + 'candy-manager', + 'candy-querier', + 'candy-store-gateway', + 'candy-disposer', + 'candy-query-frontend', + 'candy-distributor', + 'gateway', + 'overeat-gateway', + 'wrappers-api', + 'overeat_api', + 'gateway', + 'caramel-gateway', + 'licorice-admin-api', + 'popcorn-backend', + 'candy-backend', + 'almond', + 'marshmallow-querier', + ], + }, + { + name: NodeGraphDataFrameFieldNames.mainStat, + config: { + unit: 'ms/r', + displayName: 'Average response time', + }, + type: FieldType.number, + values: [ + 0.9961658833566494, + 0.6756011854064742, + 39.928285334583585, + 9.655124202033475, + 9.876837578844937, + 81.6959176729049, + 552.378420528502, + 34.02406743798451, + 2.7907819532876537, + null, + 1210.5999270997852, + 40.22305442474939, + 11.008270340421854, + 1163.33527417596, + 199.9200764328362, + 16.27026063145621, + 19.87661445917064, + 7.9747137654137195, + 20.135808118892268, + 0.22942839760131845, + 11.6566561294579, + 75.22426032984903, + 0.7326871862490554, + 10.591500091052806, + 0.43709820361586454, + 7.681333333333334, + null, + 5.743161551306568, + 22.56587650604559, + 31.38290115791745, + 11.375645375977152, + 0.5974877233841939, + 0.020271039373167783, + 8.891949445529226, + 10.293999999999995, + 20.582963262267164, + 405.7442187685195, + 8.522692621843909, + 6.924993219513745, + 34.39516266769771, + 5.952498150615941, + 0.7842759237283988, + 37.01067693570632, + 6.956706372103876, + 1.2689668998692378, + 66.66971283621828, + 13.622246129080805, + 0.9078908605310487, + 0.8179566076701271, + 1.101057295936026, + 19.887110234592154, + 0.015141602590812384, + 0.16393403903361325, + 1.753552009081408, + 1531278.5796188207, + 46.231810748849455, + 24.54774899949519, + 345061.91461907607, + 3.1331808761849147, + 2.764448605690192, + 2759.490189311903, + 0.019420568056168524, + 30.83859939288313, + 0.026999999999999993, + 2.405242467897073, + 14607.641090479348, + 11.449968675469439, + 0.33363793667426944, + 0.15786000187388566, + 0.585213874210476, + 41.39677023252635, + 0.12979907013653832, + 2.5827714948826848, + 6.868234219946023, + 21.107071160645475, + 0.053417204821597085, + 61.770970442855806, + 42.998532007278904, + 0.041043219084661645, + 217.39742114991702, + 0.21606213948511854, + 13.174790215465455, + 459.42768834463334, + 0.30539630255527017, + 447.5818111167013, + 2.18723509171844, + 3.22412985358373, + 4.200223281332062, + 16.03316977154454, + 5.830408312271875, + 1460.8431543525896, + 6.379590630002257, + 37.46723333208206, + 16.20695699632165, + 0.4593149548828589, + 2693.186775574188, + 0.034024049217002236, + 196.843, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + ], + }, + { + name: NodeGraphDataFrameFieldNames.secondaryStat, + config: { + unit: 'r/sec', + displayName: 'Requests per second', + }, + type: FieldType.number, + values: [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.03, + 0, + null, + 0, + 0, + 0, + 0, + 0.07, + 0.01, + 0, + 0, + 0.01, + 0, + 0, + 0, + 0, + 0, + 0.02, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.05, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + ], + }, + ], + meta: { preferredVisualisationType: 'nodeGraph' as const }, + name: 'edges', +}; diff --git a/public/app/plugins/panel/nodeGraph/createLayoutWorker.ts b/public/app/plugins/panel/nodeGraph/createLayoutWorker.ts index 1aa1f1e9083..c113773768d 100644 --- a/public/app/plugins/panel/nodeGraph/createLayoutWorker.ts +++ b/public/app/plugins/panel/nodeGraph/createLayoutWorker.ts @@ -1,3 +1,4 @@ import { CorsWorker as Worker } from 'app/core/utils/CorsWorker'; export const createWorker = () => new Worker(new URL('./layout.worker.js', import.meta.url)); +export const createMsaglWorker = () => new Worker(new URL('./layoutMsagl.worker.js', import.meta.url)); diff --git a/public/app/plugins/panel/nodeGraph/layout.ts b/public/app/plugins/panel/nodeGraph/layout.ts index 75878526068..75776374efe 100644 --- a/public/app/plugins/panel/nodeGraph/layout.ts +++ b/public/app/plugins/panel/nodeGraph/layout.ts @@ -3,8 +3,9 @@ import { useUnmount } from 'react-use'; import useMountedState from 'react-use/lib/useMountedState'; import { Field } from '@grafana/data'; +import { config as grafanaConfig } from '@grafana/runtime'; -import { createWorker } from './createLayoutWorker'; +import { createWorker, createMsaglWorker } from './createLayoutWorker'; import { EdgeDatum, EdgeDatumLayout, NodeDatum } from './types'; import { useNodeLimit } from './useNodeLimit'; import { graphBounds } from './utils'; @@ -83,11 +84,14 @@ export function useLayout( return; } - setLoading(true); + // Layered layout is better but also more expensive, so we switch to default force based layout for bigger graphs. + const layoutType = + grafanaConfig.featureToggles.nodeGraphDotLayout && rawNodes.length <= 500 ? 'layered' : 'default'; - // This is async but as I wanted to still run the sync grid layout and you cannot return promise from effect so + setLoading(true); + // This is async but as I wanted to still run the sync grid layout, and you cannot return promise from effect so // having callback seems ok here. - const cancel = defaultLayout(rawNodes, rawEdges, ({ nodes, edges }) => { + const cancel = layout(rawNodes, rawEdges, layoutType, ({ nodes, edges }) => { if (isMounted()) { setNodesGraph(nodes); setEdgesGraph(edges as EdgeDatumLayout[]); @@ -146,12 +150,14 @@ export function useLayout( * Wraps the layout code in a worker as it can take long and we don't want to block the main thread. * Returns a cancel function to terminate the worker. */ -function defaultLayout( +function layout( nodes: NodeDatum[], edges: EdgeDatum[], + engine: 'default' | 'layered', done: (data: { nodes: NodeDatum[]; edges: EdgeDatum[] }) => void ) { - const worker = createWorker(); + const worker = engine === 'default' ? createWorker() : createMsaglWorker(); + worker.onmessage = (event: MessageEvent<{ nodes: NodeDatum[]; edges: EdgeDatumLayout[] }>) => { for (let i = 0; i < nodes.length; i++) { // These stats needs to be Field class but the data is stringified over the worker boundary diff --git a/public/app/plugins/panel/nodeGraph/layoutMsagl.worker.js b/public/app/plugins/panel/nodeGraph/layoutMsagl.worker.js new file mode 100644 index 00000000000..f088b22f3fc --- /dev/null +++ b/public/app/plugins/panel/nodeGraph/layoutMsagl.worker.js @@ -0,0 +1,250 @@ +import { + GeomGraph, + GeomEdge, + GeomNode, + Point, + CurveFactory, + SugiyamaLayoutSettings, + LayerDirectionEnum, + layoutGeomGraph, +} from '@msagl/core'; +import { parseDot } from '@msagl/parser'; + +addEventListener('message', async (event) => { + const { nodes, edges, config } = event.data; + const [newNodes, newEdges] = layout(nodes, edges, config); + postMessage({ nodes: newNodes, edges: newEdges }); +}); + +/** + * Use d3 force layout to lay the nodes in a sensible way. This function modifies the nodes adding the x,y positions + * and also fills in node references in edges instead of node ids. + */ +export function layout(nodes, edges) { + const { mappedEdges, DOTToIdMap } = createMappings(nodes, edges); + + const dot = edgesToDOT(mappedEdges); + const graph = parseDot(dot); + const geomGraph = new GeomGraph(graph); + for (const e of graph.deepEdges) { + new GeomEdge(e); + } + + for (const n of graph.nodesBreadthFirst) { + const gn = new GeomNode(n); + gn.boundaryCurve = CurveFactory.mkCircle(50, new Point(0, 0)); + } + geomGraph.layoutSettings = new SugiyamaLayoutSettings(); + geomGraph.layoutSettings.layerDirection = LayerDirectionEnum.LR; + geomGraph.layoutSettings.LayerSeparation = 60; + geomGraph.layoutSettings.commonSettings.NodeSeparation = 40; + layoutGeomGraph(geomGraph); + + const nodesMap = {}; + for (const node of geomGraph.nodesBreadthFirst) { + nodesMap[DOTToIdMap[node.id]] = { + obj: node, + }; + } + + for (const node of nodes) { + nodesMap[node.id] = { + ...nodesMap[node.id], + datum: { + ...node, + x: nodesMap[node.id].obj.center.x, + y: nodesMap[node.id].obj.center.y, + }, + }; + } + const edgesMapped = edges.map((e) => { + return { + ...e, + source: nodesMap[e.source].datum, + target: nodesMap[e.target].datum, + }; + }); + + // This section checks if there are separate disjointed subgraphs. If so it groups nodes for each and then aligns + // each subgraph, so it starts on a single vertical line. Otherwise, they are laid out randomly from left to right. + const subgraphs = []; + for (const e of edgesMapped) { + const sourceGraph = subgraphs.find((g) => g.nodes.has(e.source)); + const targetGraph = subgraphs.find((g) => g.nodes.has(e.target)); + if (sourceGraph && targetGraph) { + // if the node sets are not the same we merge them + if (sourceGraph !== targetGraph) { + targetGraph.nodes.forEach(sourceGraph.nodes.add, sourceGraph.nodes); + subgraphs.splice(subgraphs.indexOf(targetGraph), 1); + sourceGraph.top = Math.min(sourceGraph.top, targetGraph.top); + sourceGraph.bottom = Math.max(sourceGraph.bottom, targetGraph.bottom); + sourceGraph.left = Math.min(sourceGraph.left, targetGraph.left); + sourceGraph.right = Math.max(sourceGraph.right, targetGraph.right); + } + // if the sets are the same nothing to do. + } else if (sourceGraph) { + sourceGraph.nodes.add(e.target); + sourceGraph.top = Math.min(sourceGraph.top, e.target.y); + sourceGraph.bottom = Math.max(sourceGraph.bottom, e.target.y); + sourceGraph.left = Math.min(sourceGraph.left, e.target.x); + sourceGraph.right = Math.max(sourceGraph.right, e.target.x); + } else if (targetGraph) { + targetGraph.nodes.add(e.source); + targetGraph.top = Math.min(targetGraph.top, e.source.y); + targetGraph.bottom = Math.max(targetGraph.bottom, e.source.y); + targetGraph.left = Math.min(targetGraph.left, e.source.x); + targetGraph.right = Math.max(targetGraph.right, e.source.x); + } else { + // we don't have these nodes + subgraphs.push({ + top: Math.min(e.source.y, e.target.y), + bottom: Math.max(e.source.y, e.target.y), + left: Math.min(e.source.x, e.target.x), + right: Math.max(e.source.x, e.target.x), + nodes: new Set([e.source, e.target]), + }); + } + } + + let top = 0; + let left = 0; + for (const g of subgraphs) { + if (top === 0) { + top = g.bottom + 200; + left = g.left; + } else { + const topDiff = top - g.top; + const leftDiff = left - g.left; + for (const n of g.nodes) { + n.x += leftDiff; + n.y += topDiff; + } + top += g.bottom - g.top + 200; + } + } + + const finalNodes = Object.values(nodesMap).map((v) => v.datum); + + centerNodes(finalNodes); + return [finalNodes, edgesMapped]; +} + +// We create mapping because the DOT language we use later to create the graph doesn't support arbitrary IDs. So we +// map our IDs to just an index of the node so the IDs are safe for the DOT parser and also create and inverse mapping +// for quick lookup. +function createMappings(nodes, edges) { + // Edges where the source and target IDs are the indexes we use for layout + const mappedEdges = []; + + // Key is an ID of the node and value is new ID which is just iteration index + const idToDOTMap = {}; + + // Key is an iteration index and value is actual ID of the node + const DOTToIdMap = {}; + + // Crate the maps both ways + let index = 0; + for (const edge of edges) { + if (!idToDOTMap[edge.source]) { + idToDOTMap[edge.source] = index.toString(10); + DOTToIdMap[index.toString(10)] = edge.source; + index++; + } + + if (!idToDOTMap[edge.target]) { + idToDOTMap[edge.target] = index.toString(10); + DOTToIdMap[index.toString(10)] = edge.target; + index++; + } + mappedEdges.push({ source: idToDOTMap[edge.source], target: idToDOTMap[edge.target] }); + } + + return { + mappedEdges, + DOTToIdMap, + }; +} + +function toDOT(edges, graphAttr = '', edgeAttr = '') { + let dot = ` + digraph G { + ${graphAttr} + `; + for (const edge of edges) { + dot += edge.source + '->' + edge.target + ' ' + edgeAttr + '\n'; + } + dot += nodesDOT(edges); + dot += '}'; + return dot; +} + +function edgesToDOT(edges) { + return toDOT(edges, 'rankdir="LR"; TBbalance="min"', '[ minlen=3 ]'); +} + +function nodesDOT(edges) { + let dot = ''; + const visitedNodes = new Set(); + // TODO: height/width for default sizing but nodes can have variable size now + const attr = '[fixedsize=true, width=1.2, height=1.7] \n'; + for (const edge of edges) { + if (!visitedNodes.has(edge.source)) { + dot += edge.source + attr; + } + if (!visitedNodes.has(edge.target)) { + dot += edge.target + attr; + } + } + return dot; +} + +/** + * Makes sure that the center of the graph based on its bound is in 0, 0 coordinates. + * Modifies the nodes directly. + */ +function centerNodes(nodes) { + const bounds = graphBounds(nodes); + for (let node of nodes) { + node.x = node.x - bounds.center.x; + node.y = node.y - bounds.center.y; + } +} + +/** + * Get bounds of the graph meaning the extent of the nodes in all directions. + */ +function graphBounds(nodes) { + if (nodes.length === 0) { + return { top: 0, right: 0, bottom: 0, left: 0, center: { x: 0, y: 0 } }; + } + + const bounds = nodes.reduce( + (acc, node) => { + if (node.x > acc.right) { + acc.right = node.x; + } + if (node.x < acc.left) { + acc.left = node.x; + } + if (node.y > acc.bottom) { + acc.bottom = node.y; + } + if (node.y < acc.top) { + acc.top = node.y; + } + return acc; + }, + { top: Infinity, right: -Infinity, bottom: -Infinity, left: Infinity } + ); + + const y = bounds.top + (bounds.bottom - bounds.top) / 2; + const x = bounds.left + (bounds.right - bounds.left) / 2; + + return { + ...bounds, + center: { + x, + y, + }, + }; +} diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 81937e7a826..e3c153e31cd 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -35,7 +35,8 @@ def yarn_install_step(): "name": "yarn-install", "image": images["node"], "commands": [ - "apt-get update -yq && apt-get install python -y", + # Python is needed to build `esfx`, which is needed by `msagl` + "apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 /usr/bin/python", "yarn install --immutable", ], "depends_on": [], diff --git a/scripts/webpack/webpack.common.js b/scripts/webpack/webpack.common.js index 38290858c6d..c9db4d4820a 100644 --- a/scripts/webpack/webpack.common.js +++ b/scripts/webpack/webpack.common.js @@ -127,6 +127,13 @@ module.exports = { test: /(unicons|mono|custom)[\\/].*\.svg$/, type: 'asset/source', }, + { + // Required for msagl library (used in Nodegraph panel) to work + test: /\.m?js$/, + resolve: { + fullySpecified: false, + }, + }, ], }, // https://webpack.js.org/plugins/split-chunks-plugin/#split-chunks-example-3 diff --git a/yarn.lock b/yarn.lock index 940cb901f00..0fa7eeddc78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2742,6 +2742,98 @@ __metadata: languageName: node linkType: hard +"@esfx/collection-core@npm:^1.0.0": + version: 1.0.0 + resolution: "@esfx/collection-core@npm:1.0.0" + checksum: 10/7e26d89953a201a6c965af18516e48ea0628ca2674c56c51368d61a3dc47b7d47b05984dc2392fecbbda4bb7134e735da0a01445818dd843a2b54e5b2010b4a2 + languageName: node + linkType: hard + +"@esfx/collections-hashmap@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/collections-hashmap@npm:1.0.2" + dependencies: + "@esfx/collection-core": "npm:^1.0.0" + "@esfx/equatable": "npm:^1.0.2" + checksum: 10/caef6a231ad6bb9aee06cce1a5da4918c07868be9732c96cf268762d1e7cf06365409d59c2e4d47acd3ad81022a5e552f6c1b75660ecdb2eb68265b70f67749d + languageName: node + linkType: hard + +"@esfx/collections-hashset@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/collections-hashset@npm:1.0.2" + dependencies: + "@esfx/collection-core": "npm:^1.0.0" + "@esfx/equatable": "npm:^1.0.2" + checksum: 10/75250e1682f42af85ff5e6b837e02faf27977515f52551857fcb28854025bc02d019801f062b92eef25b1781d1686453cb18726fa1468ed1c41eabe517860a4f + languageName: node + linkType: hard + +"@esfx/collections-linkedlist@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/collections-linkedlist@npm:1.0.2" + dependencies: + "@esfx/collection-core": "npm:^1.0.0" + "@esfx/equatable": "npm:^1.0.2" + checksum: 10/ddaafab0d90a8588b4292639e194217dfa5403fe4e6acc375b235ba1736a69dd43893b507f9c4aaae04377d66b23a1da0bee063be578940a03124cf0a6372096 + languageName: node + linkType: hard + +"@esfx/collections-multimap@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/collections-multimap@npm:1.0.2" + dependencies: + "@esfx/collection-core": "npm:^1.0.0" + "@esfx/collections-hashmap": "npm:^1.0.2" + "@esfx/collections-hashset": "npm:^1.0.2" + "@esfx/equatable": "npm:^1.0.2" + checksum: 10/ba6aebb8c737d9f6d9c6a06edd6904bf731c0e0eddc4aaa1fd67cf074f04441c368feee7a37af7dcab658602959d75002cc314283accf37dc1789c7a81d65914 + languageName: node + linkType: hard + +"@esfx/collections-sortedmap@npm:^1.0.0, @esfx/collections-sortedmap@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/collections-sortedmap@npm:1.0.2" + dependencies: + "@esfx/collection-core": "npm:^1.0.0" + "@esfx/equatable": "npm:^1.0.2" + checksum: 10/428f752583b5fa78e57137952f4925ca8f54daa3816a34b699adcf2206e816f3771c0bf7cfcbeafcf36e4036df200ab90bd7e6d1067bc7dc04e1dcc10f24706a + languageName: node + linkType: hard + +"@esfx/collections-sortedset@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/collections-sortedset@npm:1.0.2" + dependencies: + "@esfx/collection-core": "npm:^1.0.0" + "@esfx/equatable": "npm:^1.0.2" + checksum: 10/3f0f1fac9a861bcad3965ffa23b9519b92b6aa54bd163e72a4220cc699a7a0f8fa120deb05152fdc3a2071bd543103fe0de74760e5ce3366e3b79874b2749b1d + languageName: node + linkType: hard + +"@esfx/collections@npm:^1.0.0": + version: 1.0.2 + resolution: "@esfx/collections@npm:1.0.2" + dependencies: + "@esfx/collections-hashmap": "npm:^1.0.2" + "@esfx/collections-hashset": "npm:^1.0.2" + "@esfx/collections-linkedlist": "npm:^1.0.2" + "@esfx/collections-multimap": "npm:^1.0.2" + "@esfx/collections-sortedmap": "npm:^1.0.2" + "@esfx/collections-sortedset": "npm:^1.0.2" + checksum: 10/3f7f8d908e0e4c7ec31f48b5f52a44d3ba00fa74dc20aa2fdb811a69612ff477b2c96a288970a29073c8a7281d1cdda58614272dd1ce7d2a88305a5f98b33f9f + languageName: node + linkType: hard + +"@esfx/equatable@npm:^1.0.2": + version: 1.0.2 + resolution: "@esfx/equatable@npm:1.0.2" + dependencies: + "@mapbox/node-pre-gyp": "npm:^1.0.9" + checksum: 10/54e9de6092116699fb206676c5afd7b513e7b3e3fa03d08b8ee84a6f3b4d0057346606ceb2745f47cb1e73be28e77ca3fa403f1dcc66151a1f7a2e9cb48f7df3 + languageName: node + linkType: hard + "@eslint-community/eslint-utils@npm:^4.2.0, @eslint-community/eslint-utils@npm:^4.4.0": version: 4.4.0 resolution: "@eslint-community/eslint-utils@npm:4.4.0" @@ -3070,12 +3162,14 @@ __metadata: "@grafana/ui": "npm:10.4.0-pre" "@testing-library/react": "npm:14.2.0" "@testing-library/user-event": "npm:14.5.2" + "@types/d3-random": "npm:^3.0.2" "@types/jest": "npm:29.5.11" "@types/lodash": "npm:4.14.202" "@types/node": "npm:20.11.13" "@types/react": "npm:18.2.48" "@types/testing-library__jest-dom": "npm:5.14.9" "@types/uuid": "npm:9.0.8" + d3-random: "npm:^3.0.1" lodash: "npm:4.17.21" micro-memoize: "npm:^4.1.2" react: "npm:18.2.0" @@ -4618,6 +4712,25 @@ __metadata: languageName: node linkType: hard +"@mapbox/node-pre-gyp@npm:^1.0.9": + version: 1.0.11 + resolution: "@mapbox/node-pre-gyp@npm:1.0.11" + dependencies: + detect-libc: "npm:^2.0.0" + https-proxy-agent: "npm:^5.0.0" + make-dir: "npm:^3.1.0" + node-fetch: "npm:^2.6.7" + nopt: "npm:^5.0.0" + npmlog: "npm:^5.0.1" + rimraf: "npm:^3.0.2" + semver: "npm:^7.3.5" + tar: "npm:^6.1.11" + bin: + node-pre-gyp: bin/node-pre-gyp + checksum: 10/59529a2444e44fddb63057152452b00705aa58059079191126c79ac1388ae4565625afa84ed4dd1bf017d1111ab6e47907f7c5192e06d83c9496f2f3e708680a + languageName: node + linkType: hard + "@mapbox/point-geometry@npm:^0.1.0": version: 0.1.0 resolution: "@mapbox/point-geometry@npm:0.1.0" @@ -4677,6 +4790,42 @@ __metadata: languageName: node linkType: hard +"@msagl/core@npm:^1.1.16, @msagl/core@npm:^1.1.17": + version: 1.1.17 + resolution: "@msagl/core@npm:1.1.17" + dependencies: + "@esfx/collections": "npm:^1.0.0" + "@esfx/collections-sortedmap": "npm:^1.0.0" + queue-typescript: "npm:^1.0.1" + reliable-random: "npm:^0.0.1" + stack-typescript: "npm:^1.0.4" + typescript-string-operations: "npm:^1.4.1" + checksum: 10/6a0c83911d0802cf4372a9d976017117d9ecbec8e8bda8850fb58dec76558c0453290a51ad1c55b451ae70deeb3a57b1623bcbf7d054b3bfdacb9ea2a8190d5a + languageName: node + linkType: hard + +"@msagl/drawing@npm:^1.1.17": + version: 1.1.17 + resolution: "@msagl/drawing@npm:1.1.17" + dependencies: + "@msagl/core": "npm:^1.1.17" + checksum: 10/7a3f38cf2cafeda292cbd7c3f78d55f39f11a5923c3675ecd5a51d362660461fca50b42e5c9aa58431ad34f650fff0443be99670947c6fcd6f07701057a08c52 + languageName: node + linkType: hard + +"@msagl/parser@npm:^1.1.16": + version: 1.1.17 + resolution: "@msagl/parser@npm:1.1.17" + dependencies: + "@msagl/core": "npm:^1.1.17" + "@msagl/drawing": "npm:^1.1.17" + "@types/parse-color": "npm:^1.0.1" + dotparser: "npm:^1.1.1" + parse-color: "npm:^1.0.0" + checksum: 10/eae9c0a02410177b9c8f73edcad3c9e4610a31a972b3c58a870fe1db7c75b857304a87b82e8b7f9d6afedf91be73969856626ebb0f3fdae0d47c4e62da805ca1 + languageName: node + linkType: hard + "@mswjs/cookies@npm:^0.2.2": version: 0.2.2 resolution: "@mswjs/cookies@npm:0.2.2" @@ -8321,6 +8470,13 @@ __metadata: languageName: node linkType: hard +"@types/d3-random@npm:^3.0.2": + version: 3.0.3 + resolution: "@types/d3-random@npm:3.0.3" + checksum: 10/2c126dda6846f6c7e02c9123a30b4cdf27f3655d19b78456bbb330fbac27acceeeb987318055d3964dba8e6450377ff737db91d81f27c81ca6f4522c9b994ef2 + languageName: node + linkType: hard + "@types/d3-scale-chromatic@npm:*, @types/d3-scale-chromatic@npm:3.0.3": version: 3.0.3 resolution: "@types/d3-scale-chromatic@npm:3.0.3" @@ -8990,6 +9146,13 @@ __metadata: languageName: node linkType: hard +"@types/parse-color@npm:^1.0.1": + version: 1.0.3 + resolution: "@types/parse-color@npm:1.0.3" + checksum: 10/d9c822f820f277502cc58a4c52f5758980e6972c9a4710df515ec93dc5dde05e8e3da6210ea6514035d49c1bf36a3f720c0e5ec9c214bdc0b8c3fdb9240e36ea + languageName: node + linkType: hard + "@types/parse-json@npm:^4.0.0": version: 4.0.0 resolution: "@types/parse-json@npm:4.0.0" @@ -10792,6 +10955,16 @@ __metadata: languageName: node linkType: hard +"are-we-there-yet@npm:^2.0.0": + version: 2.0.0 + resolution: "are-we-there-yet@npm:2.0.0" + dependencies: + delegates: "npm:^1.0.0" + readable-stream: "npm:^3.6.0" + checksum: 10/ea6f47d14fc33ae9cbea3e686eeca021d9d7b9db83a306010dd04ad5f2c8b7675291b127d3fcbfcbd8fec26e47b3324ad5b469a6cc3733a582f2fe4e12fc6756 + languageName: node + linkType: hard + "are-we-there-yet@npm:^3.0.0": version: 3.0.0 resolution: "are-we-there-yet@npm:3.0.0" @@ -12535,6 +12708,13 @@ __metadata: languageName: node linkType: hard +"color-convert@npm:~0.5.0": + version: 0.5.3 + resolution: "color-convert@npm:0.5.3" + checksum: 10/00dc4256c07ed8760d7bbba234ff969c139eb964fe165853696852001002695c492e327d83ddb7a8cad8d27b49fa543d001328928c12474ee8ecb335bf5f2eb4 + languageName: node + linkType: hard + "color-name@npm:1.1.3": version: 1.1.3 resolution: "color-name@npm:1.1.3" @@ -12549,7 +12729,7 @@ __metadata: languageName: node linkType: hard -"color-support@npm:^1.1.3": +"color-support@npm:^1.1.2, color-support@npm:^1.1.3": version: 1.1.3 resolution: "color-support@npm:1.1.3" bin: @@ -13848,7 +14028,7 @@ __metadata: languageName: node linkType: hard -"d3-random@npm:3": +"d3-random@npm:3, d3-random@npm:^3.0.1": version: 3.0.1 resolution: "d3-random@npm:3.0.1" checksum: 10/9f41d6ca3a1826cea8d88392917b5039504337d442a4d1357c870fa3031701e60209a2689a6ddae7df8fca824383d038c957eb545bc49a7428c71aaf3b11f56f @@ -14380,6 +14560,13 @@ __metadata: languageName: node linkType: hard +"detect-libc@npm:^2.0.0": + version: 2.0.2 + resolution: "detect-libc@npm:2.0.2" + checksum: 10/6118f30c0c425b1e56b9d2609f29bec50d35a6af0b762b6ad127271478f3bbfda7319ce869230cf1a351f2b219f39332cde290858553336d652c77b970f15de8 + languageName: node + linkType: hard + "detect-newline@npm:^3.0.0": version: 3.1.0 resolution: "detect-newline@npm:3.1.0" @@ -14714,6 +14901,13 @@ __metadata: languageName: node linkType: hard +"dotparser@npm:^1.1.1": + version: 1.1.1 + resolution: "dotparser@npm:1.1.1" + checksum: 10/a66a02f5add34b53356ee18b418589ccf055ef6ffec073e91230f5f34cb5ac9579ce20e843b9a3213f42a26bd89fce9232b9364cc08844dade575095dc40d88a + languageName: node + linkType: hard + "duplexer@npm:^0.1.1, duplexer@npm:^0.1.2": version: 0.1.2 resolution: "duplexer@npm:0.1.2" @@ -16808,6 +17002,23 @@ __metadata: languageName: node linkType: hard +"gauge@npm:^3.0.0": + version: 3.0.2 + resolution: "gauge@npm:3.0.2" + dependencies: + aproba: "npm:^1.0.3 || ^2.0.0" + color-support: "npm:^1.1.2" + console-control-strings: "npm:^1.0.0" + has-unicode: "npm:^2.0.1" + object-assign: "npm:^4.1.1" + signal-exit: "npm:^3.0.0" + string-width: "npm:^4.2.3" + strip-ansi: "npm:^6.0.1" + wide-align: "npm:^1.1.2" + checksum: 10/46df086451672a5fecd58f7ec86da74542c795f8e00153fbef2884286ce0e86653c3eb23be2d0abb0c4a82b9b2a9dec3b09b6a1cf31c28085fa0376599a26589 + languageName: node + linkType: hard + "gauge@npm:^4.0.3": version: 4.0.4 resolution: "gauge@npm:4.0.4" @@ -17433,6 +17644,8 @@ __metadata: "@locker/near-membrane-dom": "npm:0.13.3" "@locker/near-membrane-shared": "npm:0.13.3" "@locker/near-membrane-shared-dom": "npm:0.13.3" + "@msagl/core": "npm:^1.1.16" + "@msagl/parser": "npm:^1.1.16" "@opentelemetry/api": "npm:1.7.0" "@opentelemetry/exporter-collector": "npm:0.25.0" "@opentelemetry/semantic-conventions": "npm:1.21.0" @@ -20870,6 +21083,13 @@ __metadata: languageName: node linkType: hard +"linked-list-typescript@npm:^1.0.11": + version: 1.0.15 + resolution: "linked-list-typescript@npm:1.0.15" + checksum: 10/91c87ab00fe4bb9850f169adc62b811ae7e91a743bac1a9fc969a815f20ddd0fb285f84a4d31e98900512234f59297f7beef6b076dd382cf879a319c7a14c68b + languageName: node + linkType: hard + "listr2@npm:^3.8.3": version: 3.14.0 resolution: "listr2@npm:3.14.0" @@ -21266,7 +21486,7 @@ __metadata: languageName: node linkType: hard -"make-dir@npm:^3.0.0, make-dir@npm:^3.0.2": +"make-dir@npm:^3.0.0, make-dir@npm:^3.0.2, make-dir@npm:^3.1.0": version: 3.1.0 resolution: "make-dir@npm:3.1.0" dependencies: @@ -22680,6 +22900,18 @@ __metadata: languageName: node linkType: hard +"npmlog@npm:^5.0.1": + version: 5.0.1 + resolution: "npmlog@npm:5.0.1" + dependencies: + are-we-there-yet: "npm:^2.0.0" + console-control-strings: "npm:^1.1.0" + gauge: "npm:^3.0.0" + set-blocking: "npm:^2.0.0" + checksum: 10/f42c7b9584cdd26a13c41a21930b6f5912896b6419ab15be88cc5721fc792f1c3dd30eb602b26ae08575694628ba70afdcf3675d86e4f450fc544757e52726ec + languageName: node + linkType: hard + "npmlog@npm:^6.0.0, npmlog@npm:^6.0.2": version: 6.0.2 resolution: "npmlog@npm:6.0.2" @@ -23330,6 +23562,15 @@ __metadata: languageName: node linkType: hard +"parse-color@npm:^1.0.0": + version: 1.0.0 + resolution: "parse-color@npm:1.0.0" + dependencies: + color-convert: "npm:~0.5.0" + checksum: 10/9f988462af30929acbbce135286ce20dc2fb8efe48fd0693ed8bfd7da3a60e2465cffff6d35047fe94fd7863b43159bf41b377a9982939b7c65fdba1e30af24a + languageName: node + linkType: hard + "parse-entities@npm:^2.0.0": version: 2.0.0 resolution: "parse-entities@npm:2.0.0" @@ -24582,6 +24823,15 @@ __metadata: languageName: node linkType: hard +"queue-typescript@npm:^1.0.1": + version: 1.0.1 + resolution: "queue-typescript@npm:1.0.1" + dependencies: + linked-list-typescript: "npm:^1.0.11" + checksum: 10/403e38579902af1ac533911876c31dd1fd62e94d352263b9ed9786db3677494b15236ad43182178e5103915605a8b07da5780d84b6cc075d7f54cfb056091489 + languageName: node + linkType: hard + "quick-lru@npm:^4.0.1": version: 4.0.1 resolution: "quick-lru@npm:4.0.1" @@ -26216,6 +26466,13 @@ __metadata: languageName: node linkType: hard +"reliable-random@npm:^0.0.1": + version: 0.0.1 + resolution: "reliable-random@npm:0.0.1" + checksum: 10/dbd1fe95dcea9004a0b64540a29c72832c605957a06d26eb762e6600835981ee4949ce7e15b303dcc5c44e9a976357d560f4b6f625fdfaf17f611c65a0150975 + languageName: node + linkType: hard + "remark-external-links@npm:^8.0.0": version: 8.0.0 resolution: "remark-external-links@npm:8.0.0" @@ -27778,6 +28035,15 @@ __metadata: languageName: node linkType: hard +"stack-typescript@npm:^1.0.4": + version: 1.0.4 + resolution: "stack-typescript@npm:1.0.4" + dependencies: + linked-list-typescript: "npm:^1.0.11" + checksum: 10/a3211a70b01ab4ece788a9c0c4612a132be4019427dc1d9daa7f289b5ab495d4a30cdc947888858eb9225b4c55428530ae60526c56b141f843047db1d878114e + languageName: node + linkType: hard + "stack-utils@npm:^2.0.3": version: 2.0.5 resolution: "stack-utils@npm:2.0.5" @@ -29348,6 +29614,13 @@ __metadata: languageName: node linkType: hard +"typescript-string-operations@npm:^1.4.1": + version: 1.5.0 + resolution: "typescript-string-operations@npm:1.5.0" + checksum: 10/a884c4241fc45b9cceb2744c20fb229e6dfcb48f96afd68ef2d67fda1e4586c96d20fd0343b57c14ac33b2ebe000aa7524a4932424aea3077fe9e692555384f2 + languageName: node + linkType: hard + "typescript@npm:4.8.4, typescript@npm:^4.2.4": version: 4.8.4 resolution: "typescript@npm:4.8.4" @@ -30648,7 +30921,7 @@ __metadata: languageName: node linkType: hard -"wide-align@npm:^1.1.0, wide-align@npm:^1.1.5": +"wide-align@npm:^1.1.0, wide-align@npm:^1.1.2, wide-align@npm:^1.1.5": version: 1.1.5 resolution: "wide-align@npm:1.1.5" dependencies: From e00aba0ce5a94006a07be192c5e494a79c33ccf2 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 31 Jan 2024 17:57:16 +0100 Subject: [PATCH 19/84] Prometheus: Fix OperationEditor and mounting of components (#81656) * Prometheus: Fix OperationEditor * Remove redundant import --- .../querybuilder/shared/OperationEditor.tsx | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/public/app/plugins/datasource/prometheus/querybuilder/shared/OperationEditor.tsx b/public/app/plugins/datasource/prometheus/querybuilder/shared/OperationEditor.tsx index e8630e2cdb9..36c6f213c48 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/shared/OperationEditor.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/shared/OperationEditor.tsx @@ -1,6 +1,6 @@ import { css, cx } from '@emotion/css'; import React, { useEffect, useId, useState } from 'react'; -import { Draggable, DraggableProvided } from 'react-beautiful-dnd'; +import { Draggable } from 'react-beautiful-dnd'; import { DataSourceApi, GrafanaTheme2, TimeRange } from '@grafana/data'; import { Button, Icon, InlineField, Tooltip, useTheme2, Stack } from '@grafana/ui'; @@ -145,34 +145,6 @@ export function OperationEditor({ return isConflicting ? true : undefined; }; - // We need to extract this into a component to prevent InlineField passing invalid to div which produces console error - const StyledOperationHeader = ({ provided }: { provided: DraggableProvided }) => ( -
- -
{operationElements}
- {restParam} - {index < query.operations.length - 1 && ( -
-
-
-
- )} -
- ); - return ( {(provided, snapshot) => ( @@ -181,7 +153,34 @@ export function OperationEditor({ invalid={isInvalid(snapshot.isDragging)} className={cx(styles.error, styles.cardWrapper)} > - +
+ +
{operationElements}
+ {restParam} + {index < query.operations.length - 1 && ( +
+
+
+
+ )} +
)} From c310a20966a5a7c5071ebc01af64f621870fa22c Mon Sep 17 00:00:00 2001 From: Ieva Date: Wed, 31 Jan 2024 17:09:24 +0000 Subject: [PATCH 20/84] AuthZ: add headers for IP range AC checks for cloud data sources (#80208) * add feature toggle * add a middleware that appens headers for IP range AC * sort imports * sign IP range header and only append it if the request is going to allow listed data sources * sign a random generated string instead of IP, also change the name of the middleware to make it more generic * remove the DS IP range AC options from the config file; remove unwanted change * add test * sanitize the URLs when comparing * cleanup and fixes * check if X-Real-Ip is present, and set the internal request header if it is not present * use split string function from the util package --- .../grafana_request_id_header_middleware.go | 136 ++++++++++++++++++ ...afana_request_id_header_middleware_test.go | 130 +++++++++++++++++ .../pluginsintegration/pluginsintegration.go | 4 + pkg/setting/setting.go | 14 ++ 4 files changed, 284 insertions(+) create mode 100644 pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware.go create mode 100644 pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware_test.go diff --git a/pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware.go new file mode 100644 index 00000000000..f5c62396571 --- /dev/null +++ b/pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware.go @@ -0,0 +1,136 @@ +package clientmiddleware + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "path" + + "github.com/google/uuid" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/contexthandler" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/web" +) + +const GrafanaRequestID = "X-Grafana-Request-Id" +const GrafanaSignedRequestID = "X-Grafana-Signed-Request-Id" +const GrafanaInternalRequest = "X-Grafana-Internal-Request" + +// NewHostedGrafanaACHeaderMiddleware creates a new plugins.ClientMiddleware that will +// generate a random request ID, sign it using internal key and populate X-Grafana-Request-ID with the request ID +// and X-Grafana-Signed-Request-ID with signed request ID. We can then use this to verify that the request +// is coming from hosted Grafana and is not an external request. This is used for IP range access control. +func NewHostedGrafanaACHeaderMiddleware(cfg *setting.Cfg) plugins.ClientMiddleware { + return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client { + return &HostedGrafanaACHeaderMiddleware{ + next: next, + log: log.New("ip_header_middleware"), + cfg: cfg, + } + }) +} + +type HostedGrafanaACHeaderMiddleware struct { + next plugins.Client + log log.Logger + cfg *setting.Cfg +} + +func (m *HostedGrafanaACHeaderMiddleware) applyGrafanaRequestIDHeader(ctx context.Context, pCtx backend.PluginContext, h backend.ForwardHTTPHeaders) { + // if request is not for a datasource, skip the middleware + if h == nil || pCtx.DataSourceInstanceSettings == nil { + return + } + + // Check if the request is for a datasource that is allowed to have the header + target := pCtx.DataSourceInstanceSettings.URL + + foundMatch := false + for _, allowedURL := range m.cfg.IPRangeACAllowedURLs { + if path.Clean(allowedURL) == path.Clean(target) { + foundMatch = true + break + } + } + if !foundMatch { + m.log.Debug("Data source URL not among the allow-listed URLs", "url", target) + return + } + + // Generate a new Grafana request ID and sign it with the secret key + uid, err := uuid.NewRandom() + if err != nil { + m.log.Debug("Failed to generate Grafana request ID", "error", err) + return + } + grafanaRequestID := uid.String() + + hmac := hmac.New(sha256.New, []byte(m.cfg.IPRangeACSecretKey)) + if _, err := hmac.Write([]byte(grafanaRequestID)); err != nil { + m.log.Debug("Failed to sign IP range access control header", "error", err) + return + } + signedGrafanaRequestID := hex.EncodeToString(hmac.Sum(nil)) + h.SetHTTPHeader(GrafanaSignedRequestID, signedGrafanaRequestID) + h.SetHTTPHeader(GrafanaRequestID, grafanaRequestID) + + reqCtx := contexthandler.FromContext(ctx) + if reqCtx != nil && reqCtx.Req != nil { + remoteAddress := web.RemoteAddr(reqCtx.Req) + if remoteAddress != "" { + return + } + } + h.SetHTTPHeader(GrafanaInternalRequest, "true") +} + +func (m *HostedGrafanaACHeaderMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + if req == nil { + return m.next.QueryData(ctx, req) + } + + m.applyGrafanaRequestIDHeader(ctx, req.PluginContext, req) + + return m.next.QueryData(ctx, req) +} + +func (m *HostedGrafanaACHeaderMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + if req == nil { + return m.next.CallResource(ctx, req, sender) + } + + m.applyGrafanaRequestIDHeader(ctx, req.PluginContext, req) + + return m.next.CallResource(ctx, req, sender) +} + +func (m *HostedGrafanaACHeaderMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { + if req == nil { + return m.next.CheckHealth(ctx, req) + } + + m.applyGrafanaRequestIDHeader(ctx, req.PluginContext, req) + + return m.next.CheckHealth(ctx, req) +} + +func (m *HostedGrafanaACHeaderMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { + return m.next.CollectMetrics(ctx, req) +} + +func (m *HostedGrafanaACHeaderMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { + return m.next.SubscribeStream(ctx, req) +} + +func (m *HostedGrafanaACHeaderMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { + return m.next.PublishStream(ctx, req) +} + +func (m *HostedGrafanaACHeaderMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { + return m.next.RunStream(ctx, req, sender) +} diff --git a/pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware_test.go b/pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware_test.go new file mode 100644 index 00000000000..4ef2ef76d83 --- /dev/null +++ b/pkg/services/pluginsintegration/clientmiddleware/grafana_request_id_header_middleware_test.go @@ -0,0 +1,130 @@ +package clientmiddleware + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/plugins/manager/client/clienttest" + "github.com/grafana/grafana/pkg/services/contexthandler/ctxkey" + contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/web" +) + +func Test_HostedGrafanaACHeaderMiddleware(t *testing.T) { + t.Run("Should set Grafana request ID headers if the data source URL is in the allow list", func(t *testing.T) { + cfg := setting.NewCfg() + cfg.IPRangeACAllowedURLs = []string{"https://logs.grafana.net"} + cfg.IPRangeACSecretKey = "secret" + cdt := clienttest.NewClientDecoratorTest(t, clienttest.WithMiddlewares(NewHostedGrafanaACHeaderMiddleware(cfg))) + + ctx := context.WithValue(context.Background(), ctxkey.Key{}, &contextmodel.ReqContext{ + Context: &web.Context{Req: &http.Request{ + Header: map[string][]string{"X-Real-Ip": {"1.2.3.4"}}, + }}, + SignedInUser: &user.SignedInUser{}, + }) + + err := cdt.Decorator.CallResource(ctx, &backend.CallResourceRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ + URL: "https://logs.grafana.net", + }, + }, + }, nopCallResourceSender) + require.NoError(t, err) + + require.Len(t, cdt.CallResourceReq.Headers[GrafanaRequestID], 1) + require.Len(t, cdt.CallResourceReq.Headers[GrafanaSignedRequestID], 1) + + requestID := cdt.CallResourceReq.Headers[GrafanaRequestID][0] + + instance := hmac.New(sha256.New, []byte(cfg.IPRangeACSecretKey)) + _, err = instance.Write([]byte(requestID)) + require.NoError(t, err) + computed := hex.EncodeToString(instance.Sum(nil)) + + require.Equal(t, cdt.CallResourceReq.Headers[GrafanaSignedRequestID][0], computed) + + // Internal header should not be set + require.Len(t, cdt.CallResourceReq.Headers[GrafanaInternalRequest], 0) + }) + + t.Run("Should not set Grafana request ID headers if the data source URL is not in the allow list", func(t *testing.T) { + cfg := setting.NewCfg() + cfg.IPRangeACAllowedURLs = []string{"https://logs.grafana.net"} + cfg.IPRangeACSecretKey = "secret" + cdt := clienttest.NewClientDecoratorTest(t, clienttest.WithMiddlewares(NewHostedGrafanaACHeaderMiddleware(cfg))) + + ctx := context.WithValue(context.Background(), ctxkey.Key{}, &contextmodel.ReqContext{ + Context: &web.Context{Req: &http.Request{}}, + SignedInUser: &user.SignedInUser{}, + }) + + err := cdt.Decorator.CallResource(ctx, &backend.CallResourceRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ + URL: "https://logs.not-grafana.net", + }, + }, + }, nopCallResourceSender) + require.NoError(t, err) + + require.Len(t, cdt.CallResourceReq.Headers[GrafanaRequestID], 0) + require.Len(t, cdt.CallResourceReq.Headers[GrafanaSignedRequestID], 0) + }) + + t.Run("Should set Grafana request ID headers if a sanitized data source URL is in the allow list", func(t *testing.T) { + cfg := setting.NewCfg() + cfg.IPRangeACAllowedURLs = []string{"https://logs.GRAFANA.net/"} + cfg.IPRangeACSecretKey = "secret" + cdt := clienttest.NewClientDecoratorTest(t, clienttest.WithMiddlewares(NewHostedGrafanaACHeaderMiddleware(cfg))) + + ctx := context.WithValue(context.Background(), ctxkey.Key{}, &contextmodel.ReqContext{ + Context: &web.Context{Req: &http.Request{}}, + SignedInUser: &user.SignedInUser{}, + }) + + err := cdt.Decorator.CallResource(ctx, &backend.CallResourceRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ + URL: "https://logs.grafana.net/abc/../", + }, + }, + }, nopCallResourceSender) + require.NoError(t, err) + + require.Len(t, cdt.CallResourceReq.Headers[GrafanaRequestID], 0) + require.Len(t, cdt.CallResourceReq.Headers[GrafanaSignedRequestID], 0) + }) + + t.Run("Should set Grafana internal request header if the request is internal (doesn't have X-Real-IP header set)", func(t *testing.T) { + cfg := setting.NewCfg() + cfg.IPRangeACAllowedURLs = []string{"https://logs.grafana.net"} + cfg.IPRangeACSecretKey = "secret" + cdt := clienttest.NewClientDecoratorTest(t, clienttest.WithMiddlewares(NewHostedGrafanaACHeaderMiddleware(cfg))) + + ctx := context.WithValue(context.Background(), ctxkey.Key{}, &contextmodel.ReqContext{ + Context: &web.Context{Req: &http.Request{}}, + SignedInUser: &user.SignedInUser{}, + }) + + err := cdt.Decorator.CallResource(ctx, &backend.CallResourceRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ + URL: "https://logs.grafana.net", + }, + }, + }, nopCallResourceSender) + require.NoError(t, err) + require.Equal(t, cdt.CallResourceReq.Headers[GrafanaInternalRequest][0], "true") + }) +} diff --git a/pkg/services/pluginsintegration/pluginsintegration.go b/pkg/services/pluginsintegration/pluginsintegration.go index 6edee03042d..0826cfade83 100644 --- a/pkg/services/pluginsintegration/pluginsintegration.go +++ b/pkg/services/pluginsintegration/pluginsintegration.go @@ -181,6 +181,10 @@ func CreateMiddlewares(cfg *setting.Cfg, oAuthTokenService oauthtoken.OAuthToken middlewares = append(middlewares, clientmiddleware.NewUserHeaderMiddleware()) } + if cfg.IPRangeACEnabled { + middlewares = append(middlewares, clientmiddleware.NewHostedGrafanaACHeaderMiddleware(cfg)) + } + middlewares = append(middlewares, clientmiddleware.NewHTTPClientMiddleware()) if features.IsEnabledGlobally(featuremgmt.FlagPluginsInstrumentationStatusSource) { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index ba5fc64f7a8..58bbc3b6274 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -348,6 +348,11 @@ type Cfg struct { // Number of queries to be executed concurrently. Only for the datasource supports concurrency. ConcurrentQueryCount int + // IP range access control + IPRangeACEnabled bool + IPRangeACAllowedURLs []string + IPRangeACSecretKey string + // SQL Data sources SqlDatasourceMaxOpenConnsDefault int SqlDatasourceMaxIdleConnsDefault int @@ -1200,6 +1205,7 @@ func (cfg *Cfg) parseINIFile(iniFile *ini.File) error { } cfg.readDataSourcesSettings() + cfg.readDataSourceSecuritySettings() cfg.readSqlDataSourceSettings() cfg.Storage = readStorageSettings(iniFile) @@ -1938,6 +1944,14 @@ func (cfg *Cfg) readDataSourcesSettings() { cfg.ConcurrentQueryCount = datasources.Key("concurrent_query_count").MustInt(10) } +func (cfg *Cfg) readDataSourceSecuritySettings() { + datasources := cfg.Raw.Section("datasources.ip_range_security") + cfg.IPRangeACEnabled = datasources.Key("enabled").MustBool(false) + cfg.IPRangeACSecretKey = datasources.Key("secret_key").MustString("") + allowedURLString := datasources.Key("allow_list").MustString("") + cfg.IPRangeACAllowedURLs = util.SplitString(allowedURLString) +} + func (cfg *Cfg) readSqlDataSourceSettings() { sqlDatasources := cfg.Raw.Section("sql_datasources") cfg.SqlDatasourceMaxOpenConnsDefault = sqlDatasources.Key("max_open_conns_default").MustInt(100) From 1749ec9d5e1f612c1b35dd19ecb2d73a4de0a97e Mon Sep 17 00:00:00 2001 From: Alex Khomenko Date: Wed, 31 Jan 2024 18:33:17 +0100 Subject: [PATCH 21/84] Chore: Remove Form usage from SharedPreferences.tsx (#81468) * Chore: Remove Form usage from SharedPreferences.tsx * Update betterer * Update betterer --- .betterer.results | 3 - .../SharedPreferences/SharedPreferences.tsx | 172 +++++++++--------- 2 files changed, 83 insertions(+), 92 deletions(-) diff --git a/.betterer.results b/.betterer.results index 1f976380ac4..9c7d33abd00 100644 --- a/.betterer.results +++ b/.betterer.results @@ -1050,9 +1050,6 @@ exports[`better eslint`] = { [0, 0, 0, "Use data-testid for E2E selectors instead of aria-label", "0"], [0, 0, 0, "Styles should be written using objects.", "1"] ], - "public/app/core/components/SharedPreferences/SharedPreferences.tsx:5381": [ - [0, 0, 0, "Styles should be written using objects.", "0"] - ], "public/app/core/components/TagFilter/TagFilter.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], diff --git a/public/app/core/components/SharedPreferences/SharedPreferences.tsx b/public/app/core/components/SharedPreferences/SharedPreferences.tsx index 819608c4fb1..6bdfc8c89c6 100644 --- a/public/app/core/components/SharedPreferences/SharedPreferences.tsx +++ b/public/app/core/components/SharedPreferences/SharedPreferences.tsx @@ -9,7 +9,6 @@ import { Button, Field, FieldSet, - Form, Label, Select, stylesFactory, @@ -87,7 +86,8 @@ export class SharedPreferences extends PureComponent { }); } - onSubmitForm = async () => { + onSubmitForm = async (event: React.FormEvent) => { + event.preventDefault(); const confirmationResult = this.props.onConfirm ? await this.props.onConfirm() : true; if (confirmationResult) { @@ -137,94 +137,84 @@ export class SharedPreferences extends PureComponent { const currentThemeOption = this.themeOptions.find((x) => x.value === theme) ?? this.themeOptions[0]; return ( -
- {() => { - return ( - <> -
Preferences} disabled={disabled}> - - + - - - Home Dashboard - - - } - data-testid="User preferences home dashboard drop down" - > - this.onHomeDashboardChanged(v?.uid ?? '')} - defaultOptions={true} - isClearable={true} - placeholder={t('shared-preferences.fields.home-dashboard-placeholder', 'Default dashboard')} - inputId="home-dashboard-select" - /> - + + + Home Dashboard + + + } + data-testid="User preferences home dashboard drop down" + > + this.onHomeDashboardChanged(v?.uid ?? '')} + defaultOptions={true} + isClearable={true} + placeholder={t('shared-preferences.fields.home-dashboard-placeholder', 'Default dashboard')} + inputId="home-dashboard-select" + /> + - - - + + + - - - + + + - - - Language - - - - } - data-testid="User preferences language drop down" - > - lang.value === language)} + onChange={(lang: SelectableValue) => this.onLanguageChanged(lang.value ?? '')} + options={languages} + placeholder={t('shared-preferences.fields.locale-placeholder', 'Choose language')} + inputId="locale-select" + /> + +
+ +
); } } @@ -233,9 +223,13 @@ export default SharedPreferences; const getStyles = stylesFactory(() => { return { - labelText: css` - margin-right: 6px; - `, + labelText: css({ + marginRight: '6px', + }), + form: css({ + width: '100%', + maxWidth: '600px', + }), }; }); From d1b938ba15b18040d8c5b6823ad8c82fc091df9e Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:58:02 -0600 Subject: [PATCH 22/84] Logs: Table UI - Allow users to resize field selection section (#81201) * allow user resize of fields section in table ui --------- Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --- .../explore/Logs/LogsColumnSearch.tsx | 2 +- .../features/explore/Logs/LogsTableWrap.tsx | 58 ++++++++++++++----- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/public/app/features/explore/Logs/LogsColumnSearch.tsx b/public/app/features/explore/Logs/LogsColumnSearch.tsx index 1d82f11d039..33faae16435 100644 --- a/public/app/features/explore/Logs/LogsColumnSearch.tsx +++ b/public/app/features/explore/Logs/LogsColumnSearch.tsx @@ -7,7 +7,7 @@ import { Field, Input, useTheme2 } from '@grafana/ui/src'; function getStyles(theme: GrafanaTheme2) { return { searchWrap: css({ - padding: theme.spacing(0.4), + padding: `${theme.spacing(0.4)} 0 ${theme.spacing(0.4)} ${theme.spacing(0.4)}`, }), }; } diff --git a/public/app/features/explore/Logs/LogsTableWrap.tsx b/public/app/features/explore/Logs/LogsTableWrap.tsx index 9911a50e622..9580773368a 100644 --- a/public/app/features/explore/Logs/LogsTableWrap.tsx +++ b/public/app/features/explore/Logs/LogsTableWrap.tsx @@ -1,4 +1,5 @@ import { css } from '@emotion/css'; +import { Resizable, ResizeCallback } from 're-resizable'; import React, { useCallback, useEffect, useState } from 'react'; import { @@ -265,6 +266,9 @@ export function LogsTableWrap(props: Props) { // The panel state is updated when the user interacts with the multi-select sidebar }, [currentDataFrame, getColumnsFromProps]); + const [sidebarWidth, setSidebarWidth] = useState(220); + const tableWidth = props.width - sidebarWidth; + if (!columnsWithMeta) { return null; } @@ -472,11 +476,15 @@ export function LogsTableWrap(props: Props) { props.updatePanelState({ refId: value.value, labelFieldName: logsFrame?.getLabelFieldName() ?? undefined }); }; - const sidebarWidth = 220; - const totalWidth = props.width; - const tableWidth = totalWidth - sidebarWidth; const styles = getStyles(props.theme, height, sidebarWidth); + const getOnResize: ResizeCallback = (event, direction, ref) => { + const newSidebarWidth = Number(ref.style.width.slice(0, -2)); + if (!isNaN(newSidebarWidth)) { + setSidebarWidth(newSidebarWidth); + } + }; + return ( <>
@@ -505,16 +513,24 @@ export function LogsTableWrap(props: Props) { )}
-
- - -
+ +
+ + +
+ Date: Wed, 31 Jan 2024 10:36:51 -0800 Subject: [PATCH 23/84] K8s: Add basic query service (#80325) --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/api/api.go | 2 +- pkg/api/metrics.go | 24 ++ pkg/apis/query/v0alpha1/doc.go | 6 + pkg/apis/query/v0alpha1/query.go | 211 +++++++++++ pkg/apis/query/v0alpha1/query_test.go | 75 ++++ pkg/apis/query/v0alpha1/register.go | 25 ++ pkg/apis/query/v0alpha1/results.go | 64 ++++ pkg/apis/query/v0alpha1/types.go | 65 ++++ .../query/v0alpha1/zz_generated.deepcopy.go | 140 +++++++ .../query/v0alpha1/zz_generated.defaults.go | 19 + .../query/v0alpha1/zz_generated.openapi.go | 343 ++++++++++++++++++ ...enerated.openapi_violation_exceptions.list | 5 + pkg/cmd/grafana/apiserver/server.go | 9 + .../utils.go => plugins/apiserver.go} | 6 +- .../apiserver_test.go} | 6 +- pkg/registry/apis/apis.go | 2 + pkg/registry/apis/datasource/query.go | 104 ------ pkg/registry/apis/datasource/query_test.go | 33 -- pkg/registry/apis/datasource/register.go | 6 +- pkg/registry/apis/datasource/sub_query.go | 117 +++--- pkg/registry/apis/query/parser.go | 83 +++++ pkg/registry/apis/query/plugins.go | 62 ++++ pkg/registry/apis/query/query.go | 157 ++++++++ pkg/registry/apis/query/register.go | 260 +++++++++++++ pkg/registry/apis/query/runner/direct.go | 179 +++++++++ pkg/registry/apis/query/runner/dummy.go | 88 +++++ pkg/registry/apis/wireset.go | 2 + pkg/services/featuremgmt/registry.go | 9 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/grafana-apiserver/service.go | 7 + pkg/tests/apis/datasource/testdata_test.go | 2 +- pkg/tests/apis/example/example_test.go | 4 +- pkg/tests/apis/helper.go | 41 ++- pkg/tests/apis/query/query_test.go | 98 +++++ pkg/tsdb/legacydata/conversions.go | 85 +++++ 38 files changed, 2143 insertions(+), 203 deletions(-) create mode 100644 pkg/apis/query/v0alpha1/doc.go create mode 100644 pkg/apis/query/v0alpha1/query.go create mode 100644 pkg/apis/query/v0alpha1/query_test.go create mode 100644 pkg/apis/query/v0alpha1/register.go create mode 100644 pkg/apis/query/v0alpha1/results.go create mode 100644 pkg/apis/query/v0alpha1/types.go create mode 100644 pkg/apis/query/v0alpha1/zz_generated.deepcopy.go create mode 100644 pkg/apis/query/v0alpha1/zz_generated.defaults.go create mode 100644 pkg/apis/query/v0alpha1/zz_generated.openapi.go create mode 100644 pkg/apis/query/v0alpha1/zz_generated.openapi_violation_exceptions.list rename pkg/{registry/apis/datasource/utils.go => plugins/apiserver.go} (72%) rename pkg/{registry/apis/datasource/utils_test.go => plugins/apiserver_test.go} (89%) delete mode 100644 pkg/registry/apis/datasource/query.go delete mode 100644 pkg/registry/apis/datasource/query_test.go create mode 100644 pkg/registry/apis/query/parser.go create mode 100644 pkg/registry/apis/query/plugins.go create mode 100644 pkg/registry/apis/query/query.go create mode 100644 pkg/registry/apis/query/register.go create mode 100644 pkg/registry/apis/query/runner/direct.go create mode 100644 pkg/registry/apis/query/runner/dummy.go create mode 100644 pkg/tests/apis/query/query_test.go create mode 100644 pkg/tsdb/legacydata/conversions.go diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index b92749e4304..516cf303af2 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -186,4 +186,5 @@ The following toggles require explicitly setting Grafana's [app mode]({{< relref | `externalServiceAuth` | Starts an OAuth2 authentication provider for external services | | `grafanaAPIServerWithExperimentalAPIs` | Register experimental APIs with the k8s API server | | `grafanaAPIServerEnsureKubectlAccess` | Start an additional https handler and write kubectl options | +| `kubernetesQueryServiceRewrite` | Rewrite requests targeting /ds/query to the query service | | `panelTitleSearchInV1` | Enable searching for dashboards using panel title in search v1 | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 75f54ac9821..243dc60d572 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -130,6 +130,7 @@ export interface FeatureToggles { transformationsVariableSupport?: boolean; kubernetesPlaylists?: boolean; kubernetesSnapshots?: boolean; + kubernetesQueryServiceRewrite?: boolean; cloudWatchBatchQueries?: boolean; recoveryThreshold?: boolean; lokiStructuredMetadata?: boolean; diff --git a/pkg/api/api.go b/pkg/api/api.go index 5ecc27e8029..528db6e2cab 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -515,7 +515,7 @@ func (hs *HTTPServer) registerRoutes() { // metrics // DataSource w/ expressions - apiRoute.Post("/ds/query", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.QueryMetricsV2)) + apiRoute.Post("/ds/query", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.getDSQueryEndpoint()) apiRoute.Group("/alerts", func(alertsRoute routing.RouteRegister) { alertsRoute.Post("/test", routing.Wrap(hs.AlertTest)) diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index f6acadf1358..4bbe662c081 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -10,10 +10,14 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/middleware/requestmeta" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/util/errutil/errhttp" "github.com/grafana/grafana/pkg/web" ) @@ -33,6 +37,26 @@ func (hs *HTTPServer) handleQueryMetricsError(err error) *response.NormalRespons return response.ErrOrFallback(http.StatusInternalServerError, "Query data error", err) } +// metrics.go +func (hs *HTTPServer) getDSQueryEndpoint() web.Handler { + if hs.Features.IsEnabledGlobally(featuremgmt.FlagKubernetesQueryServiceRewrite) { + // DEV ONLY FEATURE FLAG! + // rewrite requests from /ds/query to the new query service + namespaceMapper := request.GetNamespaceMapper(hs.Cfg) + return func(w http.ResponseWriter, r *http.Request) { + user, err := appcontext.User(r.Context()) + if err != nil || user == nil { + errhttp.Write(r.Context(), fmt.Errorf("no user"), w) + return + } + r.URL.Path = "/apis/query.grafana.app/v0alpha1/namespaces/" + namespaceMapper(user.OrgID) + "/query" + hs.clientConfigProvider.DirectlyServeHTTP(w, r) + } + } + + return routing.Wrap(hs.QueryMetricsV2) +} + // QueryMetricsV2 returns query metrics. // swagger:route POST /ds/query ds queryMetricsWithExpressions // diff --git a/pkg/apis/query/v0alpha1/doc.go b/pkg/apis/query/v0alpha1/doc.go new file mode 100644 index 00000000000..e6b9ec55032 --- /dev/null +++ b/pkg/apis/query/v0alpha1/doc.go @@ -0,0 +1,6 @@ +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +k8s:defaulter-gen=TypeMeta +// +groupName=query.grafana.app + +package v0alpha1 diff --git a/pkg/apis/query/v0alpha1/query.go b/pkg/apis/query/v0alpha1/query.go new file mode 100644 index 00000000000..4ae9435f0d3 --- /dev/null +++ b/pkg/apis/query/v0alpha1/query.go @@ -0,0 +1,211 @@ +package v0alpha1 + +import ( + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// Generic query request with shared time across all values +// Copied from: https://github.com/grafana/grafana/blob/main/pkg/api/dtos/models.go#L62 +type GenericQueryRequest struct { + metav1.TypeMeta `json:",inline"` + + // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now-1h + From string `json:"from,omitempty"` + + // To End time in epoch timestamps in milliseconds or relative using Grafana time units. + // example: now + To string `json:"to,omitempty"` + + // queries.refId – Specifies an identifier of the query. Is optional and default to “A”. + // queries.datasourceId – Specifies the data source to be queried. Each query in the request must have an unique datasourceId. + // queries.maxDataPoints - Species maximum amount of data points that dashboard panel can render. Is optional and default to 100. + // queries.intervalMs - Specifies the time interval in milliseconds of time series. Is optional and defaults to 1000. + // required: true + // example: [ { "refId": "A", "intervalMs": 86400000, "maxDataPoints": 1092, "datasource":{ "uid":"PD8C576611E62080A" }, "rawSql": "SELECT 1 as valueOne, 2 as valueTwo", "format": "table" } ] + Queries []GenericDataQuery `json:"queries"` + + // required: false + Debug bool `json:"debug,omitempty"` +} + +type DataSourceRef struct { + // The datasource plugin type + Type string `json:"type"` + + // Datasource UID + UID string `json:"uid"` +} + +// GenericDataQuery is a replacement for `dtos.MetricRequest` that provides more explicit types +type GenericDataQuery struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId"` + + // TimeRange represents the query range + // NOTE: unlike generic /ds/query, we can now send explicit time values in each query + TimeRange *TimeRange `json:"timeRange,omitempty"` + + // The datasource + Datasource *DataSourceRef `json:"datasource,omitempty"` + + // Deprecated -- use datasource ref instead + DatasourceId int64 `json:"datasourceId,omitempty"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType,omitempty"` + + // MaxDataPoints is the maximum number of data points that should be returned from a time series query. + MaxDataPoints int64 `json:"maxDataPoints,omitempty"` + + // Interval is the suggested duration between time points in a time series query. + IntervalMS float64 `json:"intervalMs,omitempty"` + + // true if query is disabled (ie should not be returned to the dashboard) + // Note this does not always imply that the query should not be executed since + // the results from a hidden query may be used as the input to other queries (SSE etc) + Hide bool `json:"hide,omitempty"` + + // Additional Properties (that live at the root) + props map[string]any `json:"-"` +} + +// TimeRange represents a time range for a query and is a property of DataQuery. +type TimeRange struct { + // From is the start time of the query. + From string `json:"from"` + + // To is the end time of the query. + To string `json:"to"` +} + +func (g *GenericDataQuery) AdditionalProperties() map[string]any { + if g.props == nil { + g.props = make(map[string]any) + } + return g.props +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { + *out = *g + if g.props != nil { + out.props = runtime.DeepCopyJSON(g.props) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery. +func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { + if g == nil { + return nil + } + out := new(GenericDataQuery) + g.DeepCopyInto(out) + return out +} + +// MarshalJSON ensures that the unstructured object produces proper +// JSON when passed to Go's standard JSON library. +func (g GenericDataQuery) MarshalJSON() ([]byte, error) { + vals := map[string]any{} + if g.props != nil { + for k, v := range g.props { + vals[k] = v + } + } + + vals["refId"] = g.RefID + if g.Datasource.Type != "" || g.Datasource.UID != "" { + vals["datasource"] = g.Datasource + } + if g.DatasourceId > 0 { + vals["datasourceId"] = g.DatasourceId + } + if g.IntervalMS > 0 { + vals["intervalMs"] = g.IntervalMS + } + if g.MaxDataPoints > 0 { + vals["maxDataPoints"] = g.MaxDataPoints + } + return json.Marshal(vals) +} + +// UnmarshalJSON ensures that the unstructured object properly decodes +// JSON when passed to Go's standard JSON library. +func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { + vals := map[string]any{} + err := json.Unmarshal(b, &vals) + if err != nil { + return err + } + key := "refId" + v, ok := vals[key] + if ok { + g.RefID, ok = v.(string) + if !ok { + return fmt.Errorf("expected string refid (got: %t)", v) + } + delete(vals, key) + } + + key = "datasource" + v, ok = vals[key] + if ok { + wrap, ok := v.(map[string]any) + if ok { + g.Datasource = &DataSourceRef{} + g.Datasource.Type, _ = wrap["type"].(string) + g.Datasource.UID, _ = wrap["uid"].(string) + delete(vals, key) + } else { + // Old old queries may arrive with just the name + name, ok := v.(string) + if !ok { + return fmt.Errorf("expected datasource as object (got: %t)", v) + } + g.Datasource = &DataSourceRef{} + g.Datasource.UID = name // Not great, but the lookup function will try its best to resolve + delete(vals, key) + } + } + + key = "intervalMs" + v, ok = vals[key] + if ok { + g.IntervalMS, ok = v.(float64) + if !ok { + return fmt.Errorf("expected intervalMs as float (got: %t)", v) + } + delete(vals, key) + } + + key = "maxDataPoints" + v, ok = vals[key] + if ok { + count, ok := v.(float64) + if !ok { + return fmt.Errorf("expected maxDataPoints as number (got: %t)", v) + } + g.MaxDataPoints = int64(count) + delete(vals, key) + } + + key = "datasourceId" + v, ok = vals[key] + if ok { + count, ok := v.(float64) + if !ok { + return fmt.Errorf("expected datasourceId as number (got: %t)", v) + } + g.DatasourceId = int64(count) + delete(vals, key) + } + + g.props = vals + return nil +} diff --git a/pkg/apis/query/v0alpha1/query_test.go b/pkg/apis/query/v0alpha1/query_test.go new file mode 100644 index 00000000000..e2ea3ce53c4 --- /dev/null +++ b/pkg/apis/query/v0alpha1/query_test.go @@ -0,0 +1,75 @@ +package v0alpha1_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" +) + +func TestParseQueriesIntoQueryDataRequest(t *testing.T) { + request := []byte(`{ + "queries": [ + { + "refId": "A", + "datasource": { + "type": "grafana-googlesheets-datasource", + "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" + }, + "cacheDurationSeconds": 300, + "spreadsheet": "spreadsheetID", + "datasourceId": 4, + "intervalMs": 30000, + "maxDataPoints": 794 + }, + { + "refId": "Z", + "datasource": "old", + "maxDataPoints": 10, + "timeRange": { + "from": "100", + "to": "200" + } + } + ], + "from": "1692624667389", + "to": "1692646267389" + }`) + + req := &v0alpha1.GenericQueryRequest{} + err := json.Unmarshal(request, req) + require.NoError(t, err) + + require.Len(t, req.Queries, 2) + require.Equal(t, "b1808c48-9fc9-4045-82d7-081781f8a553", req.Queries[0].Datasource.UID) + require.Equal(t, "spreadsheetID", req.Queries[0].AdditionalProperties()["spreadsheet"]) + + // Write the query (with additional spreadsheetID) to JSON + out, err := json.MarshalIndent(req.Queries[0], "", " ") + require.NoError(t, err) + + // And read it back with standard JSON marshal functions + query := &v0alpha1.GenericDataQuery{} + err = json.Unmarshal(out, query) + require.NoError(t, err) + require.Equal(t, "spreadsheetID", query.AdditionalProperties()["spreadsheet"]) + + // The second query has an explicit time range, and legacy datasource name + out, err = json.MarshalIndent(req.Queries[1], "", " ") + require.NoError(t, err) + // fmt.Printf("%s\n", string(out)) + require.JSONEq(t, `{ + "datasource": { + "type": "", ` /* NOTE! this implies legacy naming */ +` + "uid": "old" + }, + "maxDataPoints": 10, + "refId": "Z", + "timeRange": { + "from": "100", + "to": "200" + } + }`, string(out)) +} diff --git a/pkg/apis/query/v0alpha1/register.go b/pkg/apis/query/v0alpha1/register.go new file mode 100644 index 00000000000..b728ad7b43a --- /dev/null +++ b/pkg/apis/query/v0alpha1/register.go @@ -0,0 +1,25 @@ +package v0alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" +) + +const ( + GROUP = "query.grafana.app" + VERSION = "v0alpha1" + APIVERSION = GROUP + "/" + VERSION +) + +var DataSourceApiServerResourceInfo = common.NewResourceInfo(GROUP, VERSION, + "datasourceapiservers", "datasourceapiserver", "DataSourceApiServer", + func() runtime.Object { return &DataSourceApiServer{} }, + func() runtime.Object { return &DataSourceApiServerList{} }, +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: GROUP, Version: VERSION} +) diff --git a/pkg/apis/query/v0alpha1/results.go b/pkg/apis/query/v0alpha1/results.go new file mode 100644 index 00000000000..2c3cab7b3f0 --- /dev/null +++ b/pkg/apis/query/v0alpha1/results.go @@ -0,0 +1,64 @@ +package v0alpha1 + +import ( + "encoding/json" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + openapi "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +// Wraps backend.QueryDataResponse, however it includes TypeMeta and implements runtime.Object +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type QueryDataResponse struct { + metav1.TypeMeta `json:",inline"` + + // Backend wrapper (external dependency) + backend.QueryDataResponse +} + +// Expose backend DataResponse in OpenAPI (yes this still requires some serious love!) +func (r QueryDataResponse) OpenAPIDefinition() openapi.OpenAPIDefinition { + return openapi.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Allows: true}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: map[string]interface{}{ + "x-kubernetes-preserve-unknown-fields": true, + }, + }, + }, + } +} + +// MarshalJSON writes the results as json +func (r QueryDataResponse) MarshalJSON() ([]byte, error) { + return r.QueryDataResponse.MarshalJSON() +} + +// UnmarshalJSON will read JSON into a QueryDataResponse +func (r *QueryDataResponse) UnmarshalJSON(b []byte) error { + return r.QueryDataResponse.UnmarshalJSON(b) +} + +func (r *QueryDataResponse) DeepCopy() *QueryDataResponse { + if r == nil { + return nil + } + + // /!\ The most dumb approach, but OK for now... + // likely best to move DeepCopy into SDK + out := &QueryDataResponse{} + body, _ := json.Marshal(r.QueryDataResponse) + _ = json.Unmarshal(body, &out.QueryDataResponse) + return out +} + +func (r *QueryDataResponse) DeepCopyInto(out *QueryDataResponse) { + clone := r.DeepCopy() + *out = *clone +} diff --git a/pkg/apis/query/v0alpha1/types.go b/pkg/apis/query/v0alpha1/types.go new file mode 100644 index 00000000000..3f288eee25d --- /dev/null +++ b/pkg/apis/query/v0alpha1/types.go @@ -0,0 +1,65 @@ +package v0alpha1 + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// The query runner interface +type QueryRunner interface { + // Runs the query as the user in context + ExecuteQueryData(ctx context.Context, + // The k8s group for the datasource (pluginId) + datasource schema.GroupVersion, + + // The datasource name/uid + name string, + + // The raw backend query objects + query []GenericDataQuery, + ) (*backend.QueryDataResponse, error) +} + +type DataSourceApiServerRegistry interface { + // Get the group and preferred version for a plugin + GetDatasourceGroupVersion(pluginId string) (schema.GroupVersion, error) + + // Get the list of available datasource api servers + // The values will be managed though API discovery/reconciliation + GetDatasourceApiServers(ctx context.Context) (*DataSourceApiServerList, error) +} + +// The data source resource is a reflection of the individual datasource instances +// that are exposed in the groups: {datasource}.datasource.grafana.app +// The status is updated periodically. +// The name is the plugin id +// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DataSourceApiServer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // The display name + Title string `json:"title"` + + // Describe the plugin + Description string `json:"description,omitempty"` + + // The group + preferred version + GroupVersion string `json:"groupVersion"` + + // Possible alternative plugin IDs + AliasIDs []string `json:"aliasIDs,omitempty"` +} + +// List of datasource plugins +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DataSourceApiServerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []DataSourceApiServer `json:"items,omitempty"` +} diff --git a/pkg/apis/query/v0alpha1/zz_generated.deepcopy.go b/pkg/apis/query/v0alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..927ed373ddc --- /dev/null +++ b/pkg/apis/query/v0alpha1/zz_generated.deepcopy.go @@ -0,0 +1,140 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v0alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSourceApiServer) DeepCopyInto(out *DataSourceApiServer) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.AliasIDs != nil { + in, out := &in.AliasIDs, &out.AliasIDs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSourceApiServer. +func (in *DataSourceApiServer) DeepCopy() *DataSourceApiServer { + if in == nil { + return nil + } + out := new(DataSourceApiServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataSourceApiServer) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSourceApiServerList) DeepCopyInto(out *DataSourceApiServerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DataSourceApiServer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSourceApiServerList. +func (in *DataSourceApiServerList) DeepCopy() *DataSourceApiServerList { + if in == nil { + return nil + } + out := new(DataSourceApiServerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataSourceApiServerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSourceRef) DeepCopyInto(out *DataSourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSourceRef. +func (in *DataSourceRef) DeepCopy() *DataSourceRef { + if in == nil { + return nil + } + out := new(DataSourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GenericQueryRequest) DeepCopyInto(out *GenericQueryRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Queries != nil { + in, out := &in.Queries, &out.Queries + *out = make([]GenericDataQuery, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericQueryRequest. +func (in *GenericQueryRequest) DeepCopy() *GenericQueryRequest { + if in == nil { + return nil + } + out := new(GenericQueryRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *QueryDataResponse) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimeRange) DeepCopyInto(out *TimeRange) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeRange. +func (in *TimeRange) DeepCopy() *TimeRange { + if in == nil { + return nil + } + out := new(TimeRange) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/query/v0alpha1/zz_generated.defaults.go b/pkg/apis/query/v0alpha1/zz_generated.defaults.go new file mode 100644 index 00000000000..238fc2f4edc --- /dev/null +++ b/pkg/apis/query/v0alpha1/zz_generated.defaults.go @@ -0,0 +1,19 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v0alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/pkg/apis/query/v0alpha1/zz_generated.openapi.go b/pkg/apis/query/v0alpha1/zz_generated.openapi.go new file mode 100644 index 00000000000..c3a66de455c --- /dev/null +++ b/pkg/apis/query/v0alpha1/zz_generated.openapi.go @@ -0,0 +1,343 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by openapi-gen. DO NOT EDIT. + +// This file was autogenerated by openapi-gen. Do not edit it manually! + +package v0alpha1 + +import ( + common "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + return map[string]common.OpenAPIDefinition{ + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceApiServer": schema_pkg_apis_query_v0alpha1_DataSourceApiServer(ref), + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceApiServerList": schema_pkg_apis_query_v0alpha1_DataSourceApiServerList(ref), + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceRef": schema_pkg_apis_query_v0alpha1_DataSourceRef(ref), + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.GenericDataQuery": schema_pkg_apis_query_v0alpha1_GenericDataQuery(ref), + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.GenericQueryRequest": schema_pkg_apis_query_v0alpha1_GenericQueryRequest(ref), + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryDataResponse": QueryDataResponse{}.OpenAPIDefinition(), + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.TimeRange": schema_pkg_apis_query_v0alpha1_TimeRange(ref), + } +} + +func schema_pkg_apis_query_v0alpha1_DataSourceApiServer(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "The data source resource is a reflection of the individual datasource instances that are exposed in the groups: {datasource}.datasource.grafana.app The status is updated periodically. The name is the plugin id", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "title": { + SchemaProps: spec.SchemaProps{ + Description: "The display name", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "description": { + SchemaProps: spec.SchemaProps{ + Description: "Describe the plugin", + Type: []string{"string"}, + Format: "", + }, + }, + "groupVersion": { + SchemaProps: spec.SchemaProps{ + Description: "The group + preferred version", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "aliasIDs": { + SchemaProps: spec.SchemaProps{ + Description: "Possible alternative plugin IDs", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + Required: []string{"title", "groupVersion"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_query_v0alpha1_DataSourceApiServerList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "List of datasource plugins", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceApiServer"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceApiServer", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"}, + } +} + +func schema_pkg_apis_query_v0alpha1_DataSourceRef(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Description: "The datasource plugin type", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "uid": { + SchemaProps: spec.SchemaProps{ + Description: "Datasource UID", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"type", "uid"}, + }, + }, + } +} + +func schema_pkg_apis_query_v0alpha1_GenericDataQuery(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GenericDataQuery is a replacement for `dtos.MetricRequest` that provides more explicit types", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "refId": { + SchemaProps: spec.SchemaProps{ + Description: "RefID is the unique identifier of the query, set by the frontend call.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "timeRange": { + SchemaProps: spec.SchemaProps{ + Description: "TimeRange represents the query range NOTE: unlike generic /ds/query, we can now send explicit time values in each query", + Ref: ref("github.com/grafana/grafana/pkg/apis/query/v0alpha1.TimeRange"), + }, + }, + "datasource": { + SchemaProps: spec.SchemaProps{ + Description: "The datasource", + Ref: ref("github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceRef"), + }, + }, + "datasourceId": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated -- use datasource ref instead", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "queryType": { + SchemaProps: spec.SchemaProps{ + Description: "QueryType is an optional identifier for the type of query. It can be used to distinguish different types of queries.", + Type: []string{"string"}, + Format: "", + }, + }, + "maxDataPoints": { + SchemaProps: spec.SchemaProps{ + Description: "MaxDataPoints is the maximum number of data points that should be returned from a time series query.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "intervalMs": { + SchemaProps: spec.SchemaProps{ + Description: "Interval is the suggested duration between time points in a time series query.", + Type: []string{"number"}, + Format: "double", + }, + }, + "hide": { + SchemaProps: spec.SchemaProps{ + Description: "true if query is disabled (ie should not be returned to the dashboard) Note this does not always imply that the query should not be executed since the results from a hidden query may be used as the input to other queries (SSE etc)", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"refId"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.DataSourceRef", "github.com/grafana/grafana/pkg/apis/query/v0alpha1.TimeRange"}, + } +} + +func schema_pkg_apis_query_v0alpha1_GenericQueryRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Generic query request with shared time across all values Copied from: https://github.com/grafana/grafana/blob/main/pkg/api/dtos/models.go#L62", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "from": { + SchemaProps: spec.SchemaProps{ + Description: "From Start time in epoch timestamps in milliseconds or relative using Grafana time units. example: now-1h", + Type: []string{"string"}, + Format: "", + }, + }, + "to": { + SchemaProps: spec.SchemaProps{ + Description: "To End time in epoch timestamps in milliseconds or relative using Grafana time units. example: now", + Type: []string{"string"}, + Format: "", + }, + }, + "queries": { + SchemaProps: spec.SchemaProps{ + Description: "queries.refId – Specifies an identifier of the query. Is optional and default to “A”. queries.datasourceId – Specifies the data source to be queried. Each query in the request must have an unique datasourceId. queries.maxDataPoints - Species maximum amount of data points that dashboard panel can render. Is optional and default to 100. queries.intervalMs - Specifies the time interval in milliseconds of time series. Is optional and defaults to 1000. required: true example: [ { \"refId\": \"A\", \"intervalMs\": 86400000, \"maxDataPoints\": 1092, \"datasource\":{ \"uid\":\"PD8C576611E62080A\" }, \"rawSql\": \"SELECT 1 as valueOne, 2 as valueTwo\", \"format\": \"table\" } ]", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/grafana/grafana/pkg/apis/query/v0alpha1.GenericDataQuery"), + }, + }, + }, + }, + }, + "debug": { + SchemaProps: spec.SchemaProps{ + Description: "required: false", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"queries"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/query/v0alpha1.GenericDataQuery"}, + } +} + +func schema_pkg_apis_query_v0alpha1_TimeRange(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TimeRange represents a time range for a query and is a property of DataQuery.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "from": { + SchemaProps: spec.SchemaProps{ + Description: "From is the start time of the query.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "to": { + SchemaProps: spec.SchemaProps{ + Description: "To is the end time of the query.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"from", "to"}, + }, + }, + } +} diff --git a/pkg/apis/query/v0alpha1/zz_generated.openapi_violation_exceptions.list b/pkg/apis/query/v0alpha1/zz_generated.openapi_violation_exceptions.list new file mode 100644 index 00000000000..b6d2a25a924 --- /dev/null +++ b/pkg/apis/query/v0alpha1/zz_generated.openapi_violation_exceptions.list @@ -0,0 +1,5 @@ +API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/query/v0alpha1,DataSourceApiServer,AliasIDs +API rule violation: list_type_missing,github.com/grafana/grafana/pkg/apis/query/v0alpha1,GenericQueryRequest,Queries +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/query/v0alpha1,GenericDataQuery,IntervalMS +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/query/v0alpha1,GenericDataQuery,RefID +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/query/v0alpha1,QueryDataResponse,QueryDataResponse diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index 447567bb3de..d40f89d615b 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -14,6 +14,8 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/example" "github.com/grafana/grafana/pkg/registry/apis/featuretoggle" + "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/query/runner" "github.com/grafana/grafana/pkg/server" "github.com/grafana/grafana/pkg/services/featuremgmt" grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver" @@ -50,6 +52,13 @@ func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error { // No dependencies for testing case "example.grafana.app": o.builders = append(o.builders, example.NewTestingAPIBuilder()) + // Only works with testdata + case "query.grafana.app": + o.builders = append(o.builders, query.NewQueryAPIBuilder( + featuremgmt.WithFeatures(), + runner.NewDummyTestRunner(), + runner.NewDummyRegistry(), + )) case "featuretoggle.grafana.app": features := featuremgmt.WithFeatureManager(setting.FeatureMgmtSettings{}, nil) // none... for now o.builders = append(o.builders, featuretoggle.NewFeatureFlagAPIBuilder(features)) diff --git a/pkg/registry/apis/datasource/utils.go b/pkg/plugins/apiserver.go similarity index 72% rename from pkg/registry/apis/datasource/utils.go rename to pkg/plugins/apiserver.go index 5beca00d70b..e72355beb9e 100644 --- a/pkg/registry/apis/datasource/utils.go +++ b/pkg/plugins/apiserver.go @@ -1,11 +1,13 @@ -package datasource +package plugins import ( "fmt" "strings" ) -func getDatasourceGroupNameFromPluginID(pluginId string) (string, error) { +// Get the default API group name for from a plugin ID +// NOTE: this is a work in progress, and may change without notice +func GetDatasourceGroupNameFromPluginID(pluginId string) (string, error) { if pluginId == "" { return "", fmt.Errorf("bad pluginID (empty)") } diff --git a/pkg/registry/apis/datasource/utils_test.go b/pkg/plugins/apiserver_test.go similarity index 89% rename from pkg/registry/apis/datasource/utils_test.go rename to pkg/plugins/apiserver_test.go index 2bfded3d4e7..24d7102a40e 100644 --- a/pkg/registry/apis/datasource/utils_test.go +++ b/pkg/plugins/apiserver_test.go @@ -1,4 +1,4 @@ -package datasource +package plugins import ( "testing" @@ -22,11 +22,11 @@ func TestUtils(t *testing.T) { } func getIDIgnoreError(id string) string { - v, _ := getDatasourceGroupNameFromPluginID(id) + v, _ := GetDatasourceGroupNameFromPluginID(id) return v } func getErrorIgnoreValue(id string) error { - _, err := getDatasourceGroupNameFromPluginID(id) + _, err := GetDatasourceGroupNameFromPluginID(id) return err } diff --git a/pkg/registry/apis/apis.go b/pkg/registry/apis/apis.go index c2575481d70..f83ac018bda 100644 --- a/pkg/registry/apis/apis.go +++ b/pkg/registry/apis/apis.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/featuretoggle" "github.com/grafana/grafana/pkg/registry/apis/folders" "github.com/grafana/grafana/pkg/registry/apis/playlist" + "github.com/grafana/grafana/pkg/registry/apis/query" ) var ( @@ -27,6 +28,7 @@ func ProvideRegistryServiceSink( _ *featuretoggle.FeatureFlagAPIBuilder, _ *datasource.DataSourceAPIBuilder, _ *folders.FolderAPIBuilder, + _ *query.QueryAPIBuilder, ) *Service { return &Service{} } diff --git a/pkg/registry/apis/datasource/query.go b/pkg/registry/apis/datasource/query.go deleted file mode 100644 index 7952bcd561f..00000000000 --- a/pkg/registry/apis/datasource/query.go +++ /dev/null @@ -1,104 +0,0 @@ -package datasource - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - - "github.com/grafana/grafana/pkg/tsdb/legacydata" -) - -// Copied from: https://github.com/grafana/grafana/blob/main/pkg/api/dtos/models.go#L62 -type rawMetricRequest struct { - // From Start time in epoch timestamps in milliseconds or relative using Grafana time units. - // required: true - // example: now-1h - From string `json:"from"` - // To End time in epoch timestamps in milliseconds or relative using Grafana time units. - // required: true - // example: now - To string `json:"to"` - // queries.refId – Specifies an identifier of the query. Is optional and default to “A”. - // queries.datasourceId – Specifies the data source to be queried. Each query in the request must have an unique datasourceId. - // queries.maxDataPoints - Species maximum amount of data points that dashboard panel can render. Is optional and default to 100. - // queries.intervalMs - Specifies the time interval in milliseconds of time series. Is optional and defaults to 1000. - // required: true - // example: [ { "refId": "A", "intervalMs": 86400000, "maxDataPoints": 1092, "datasource":{ "uid":"PD8C576611E62080A" }, "rawSql": "SELECT 1 as valueOne, 2 as valueTwo", "format": "table" } ] - Queries []rawDataQuery `json:"queries"` - // required: false - Debug bool `json:"debug"` -} - -type rawDataQuery = map[string]interface{} - -func readQueries(in []byte) ([]backend.DataQuery, error) { - reqDTO := &rawMetricRequest{} - err := json.Unmarshal(in, &reqDTO) - if err != nil { - return nil, err - } - - if len(reqDTO.Queries) == 0 { - return nil, fmt.Errorf("expected queries") - } - - tr := legacydata.NewDataTimeRange(reqDTO.From, reqDTO.To) - backendTr := backend.TimeRange{ - From: tr.MustGetFrom(), - To: tr.MustGetTo(), - } - queries := make([]backend.DataQuery, 0) - - for _, query := range reqDTO.Queries { - dataQuery := backend.DataQuery{ - TimeRange: backendTr, - } - - v, ok := query["refId"] - if ok { - dataQuery.RefID, ok = v.(string) - if !ok { - return nil, fmt.Errorf("expeted string refId") - } - } - - v, ok = query["queryType"] - if ok { - dataQuery.QueryType, ok = v.(string) - if !ok { - return nil, fmt.Errorf("expeted string queryType") - } - } - - v, ok = query["maxDataPoints"] - if ok { - vInt, ok := v.(float64) - if !ok { - return nil, fmt.Errorf("expected float64 maxDataPoints") - } - - dataQuery.MaxDataPoints = int64(vInt) - } - - v, ok = query["intervalMs"] - if ok { - vInt, ok := v.(float64) - if !ok { - return nil, fmt.Errorf("expected float64 intervalMs") - } - - dataQuery.Interval = time.Duration(vInt) - } - - dataQuery.JSON, err = json.Marshal(query) - if err != nil { - return nil, err - } - - queries = append(queries, dataQuery) - } - - return queries, nil -} diff --git a/pkg/registry/apis/datasource/query_test.go b/pkg/registry/apis/datasource/query_test.go deleted file mode 100644 index 662bbd2e909..00000000000 --- a/pkg/registry/apis/datasource/query_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package datasource - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestParseQueriesIntoQueryDataRequest(t *testing.T) { - request := []byte(`{ - "queries": [ - { - "refId": "A", - "datasource": { - "type": "grafana-googlesheets-datasource", - "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" - }, - "cacheDurationSeconds": 300, - "spreadsheet": "spreadsheetID", - "range": "", - "datasourceId": 4, - "intervalMs": 30000, - "maxDataPoints": 794 - } - ], - "from": "1692624667389", - "to": "1692646267389" - }`) - - parsedDataQuery, err := readQueries(request) - require.NoError(t, err) - require.Equal(t, len(parsedDataQuery), 1) -} diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index 86769661890..9fc43b7bc38 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -20,6 +20,7 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" + query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -112,7 +113,8 @@ func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { &v0alpha1.DataSourceConnectionList{}, &v0alpha1.HealthCheckResult{}, &unstructured.Unstructured{}, - // Added for subresource stubs + // Query handler + &query.QueryDataResponse{}, &metav1.Status{}, ) } @@ -138,7 +140,7 @@ func (b *DataSourceAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { } func resourceFromPluginID(pluginID string) (common.ResourceInfo, error) { - group, err := getDatasourceGroupNameFromPluginID(pluginID) + group, err := plugins.GetDatasourceGroupNameFromPluginID(pluginID) if err != nil { return common.ResourceInfo{}, err } diff --git a/pkg/registry/apis/datasource/sub_query.go b/pkg/registry/apis/datasource/sub_query.go index 57ee6a58d4f..14c4c410bfb 100644 --- a/pkg/registry/apis/datasource/sub_query.go +++ b/pkg/registry/apis/datasource/sub_query.go @@ -3,14 +3,18 @@ package datasource import ( "context" "encoding/json" - "io" + "fmt" "net/http" - "time" + "strconv" "github.com/grafana/grafana-plugin-sdk-go/backend" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/middleware/requestmeta" + "github.com/grafana/grafana/pkg/tsdb/legacydata" + "github.com/grafana/grafana/pkg/web" ) type subQueryREST struct { @@ -20,11 +24,10 @@ type subQueryREST struct { var _ = rest.Connecter(&subQueryREST{}) func (r *subQueryREST) New() runtime.Object { - return &metav1.Status{} + return &v0alpha1.QueryDataResponse{} } -func (r *subQueryREST) Destroy() { -} +func (r *subQueryREST) Destroy() {} func (r *subQueryREST) ConnectMethods() []string { return []string{"POST", "GET"} @@ -34,40 +37,39 @@ func (r *subQueryREST) NewConnectOptions() (runtime.Object, bool, string) { return nil, false, "" } -func (r *subQueryREST) readQueries(req *http.Request) ([]backend.DataQuery, error) { +func (r *subQueryREST) readQueries(req *http.Request) ([]backend.DataQuery, *v0alpha1.DataSourceRef, error) { + reqDTO := v0alpha1.GenericQueryRequest{} // Simple URL to JSON mapping if req.Method == http.MethodGet { - body := make(map[string]any, 0) - for k, v := range req.URL.Query() { - switch len(v) { - case 0: - body[k] = true - case 1: - body[k] = v[0] // TODO, convert numbers + query := v0alpha1.GenericDataQuery{ + RefID: "A", + MaxDataPoints: 1000, + IntervalMS: 10, + } + params := req.URL.Query() + for k := range params { + v := params.Get(k) // the singular value + switch k { + case "to": + reqDTO.To = v + case "from": + reqDTO.From = v + case "maxDataPoints": + query.MaxDataPoints, _ = strconv.ParseInt(v, 10, 64) + case "intervalMs": + query.IntervalMS, _ = strconv.ParseFloat(v, 64) + case "queryType": + query.QueryType = v default: - body[k] = v // TODO, convert numbers + query.AdditionalProperties()[k] = v } } - - var err error - dq := backend.DataQuery{ - RefID: "A", - TimeRange: backend.TimeRange{ - From: time.Now().Add(-1 * time.Hour), // last hour - To: time.Now(), - }, - MaxDataPoints: 1000, - Interval: time.Second * 10, - } - dq.JSON, err = json.Marshal(body) - return []backend.DataQuery{dq}, err + reqDTO.Queries = []v0alpha1.GenericDataQuery{query} + } else if err := web.Bind(req, &reqDTO); err != nil { + return nil, nil, err } - body, err := io.ReadAll(req.Body) - if err != nil { - return nil, err - } - return readQueries(body) + return legacydata.ToDataSourceQueries(reqDTO) } func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) { @@ -77,27 +79,56 @@ func (r *subQueryREST) Connect(ctx context.Context, name string, opts runtime.Ob } return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - queries, err := r.readQueries(req) + queries, dsRef, err := r.readQueries(req) if err != nil { responder.Error(err) return } + if dsRef != nil && dsRef.UID != name { + responder.Error(fmt.Errorf("expected the datasource in the request url and body to match")) + return + } - queryResponse, err := r.builder.client.QueryData(ctx, &backend.QueryDataRequest{ + qdr, err := r.builder.client.QueryData(ctx, &backend.QueryDataRequest{ PluginContext: pluginCtx, Queries: queries, - // Headers: // from context }) - if err != nil { - return - } - - jsonRsp, err := json.Marshal(queryResponse) if err != nil { responder.Error(err) return } - w.WriteHeader(200) - _, _ = w.Write(jsonRsp) + + statusCode := http.StatusOK + for _, res := range qdr.Responses { + if res.Error != nil { + statusCode = http.StatusMultiStatus + } + } + if statusCode != http.StatusOK { + requestmeta.WithDownstreamStatusSource(ctx) + } + + // TODO... someday :) can return protobuf for machine-machine communication + // will avoid some hops the current response workflow (for external plugins) + // 1. Plugin: + // creates: golang structs + // returns: arrow + protobuf | + // 2. Client: | direct when local/non grpc + // reads: protobuf+arrow V + // returns: golang structs + // 3. Datasource Server (eg right here): + // reads: golang structs + // returns: JSON + // 4. Query service (alerting etc): + // reads: JSON? (TODO! raw output from 1???) + // returns: JSON (after more operations) + // 5. Browser + // reads: JSON + w.WriteHeader(statusCode) + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(qdr) + if err != nil { + responder.Error(err) + } }), nil } diff --git a/pkg/registry/apis/query/parser.go b/pkg/registry/apis/query/parser.go new file mode 100644 index 00000000000..bc60a5b451c --- /dev/null +++ b/pkg/registry/apis/query/parser.go @@ -0,0 +1,83 @@ +package query + +import ( + "fmt" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/expr" +) + +type parsedQueryRequest struct { + // The queries broken into requests + Requests []groupedQueries + + // Optionally show the additional query properties + Expressions []v0alpha1.GenericDataQuery +} + +type groupedQueries struct { + // the plugin type + pluginId string + + // The datasource name/uid + uid string + + // The raw backend query objects + query []v0alpha1.GenericDataQuery +} + +// Internally define what makes this request unique (eventually may include the apiVersion) +func (d *groupedQueries) key() string { + return fmt.Sprintf("%s/%s", d.pluginId, d.uid) +} + +func parseQueryRequest(raw v0alpha1.GenericQueryRequest) (parsedQueryRequest, error) { + mixed := make(map[string]*groupedQueries) + parsed := parsedQueryRequest{} + refIds := make(map[string]bool) + + for _, original := range raw.Queries { + if refIds[original.RefID] { + return parsed, fmt.Errorf("invalid query, duplicate refId: " + original.RefID) + } + + refIds[original.RefID] = true + q := original + + if q.TimeRange == nil && raw.From != "" { + q.TimeRange = &v0alpha1.TimeRange{ + From: raw.From, + To: raw.To, + } + } + + // Extract out the expressions queries earlier + if expr.IsDataSource(q.Datasource.Type) || expr.IsDataSource(q.Datasource.UID) { + parsed.Expressions = append(parsed.Expressions, q) + continue + } + + g := &groupedQueries{pluginId: q.Datasource.Type, uid: q.Datasource.UID} + group, ok := mixed[g.key()] + if !ok || group == nil { + group = g + mixed[g.key()] = g + } + group.query = append(group.query, q) + } + + for _, q := range parsed.Expressions { + // TODO: parse and build tree, for now just fail fast on unknown commands + _, err := expr.GetExpressionCommandType(q.AdditionalProperties()) + if err != nil { + return parsed, err + } + } + + // Add each request + for _, v := range mixed { + parsed.Requests = append(parsed.Requests, *v) + } + + return parsed, nil +} diff --git a/pkg/registry/apis/query/plugins.go b/pkg/registry/apis/query/plugins.go new file mode 100644 index 00000000000..786ba1a5c60 --- /dev/null +++ b/pkg/registry/apis/query/plugins.go @@ -0,0 +1,62 @@ +package query + +import ( + "context" + + "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" + query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" +) + +var ( + _ rest.Storage = (*pluginsStorage)(nil) + _ rest.Scoper = (*pluginsStorage)(nil) + _ rest.SingularNameProvider = (*pluginsStorage)(nil) + _ rest.Lister = (*pluginsStorage)(nil) +) + +type pluginsStorage struct { + resourceInfo *common.ResourceInfo + tableConverter rest.TableConvertor + registry query.DataSourceApiServerRegistry +} + +func newPluginsStorage(reg query.DataSourceApiServerRegistry) *pluginsStorage { + var resourceInfo = query.DataSourceApiServerResourceInfo + return &pluginsStorage{ + resourceInfo: &resourceInfo, + tableConverter: rest.NewDefaultTableConvertor(resourceInfo.GroupResource()), + registry: reg, + } +} + +func (s *pluginsStorage) New() runtime.Object { + return s.resourceInfo.NewFunc() +} + +func (s *pluginsStorage) Destroy() {} + +func (s *pluginsStorage) NamespaceScoped() bool { + return false +} + +func (s *pluginsStorage) GetSingularName() string { + return example.DummyResourceInfo.GetSingularName() +} + +func (s *pluginsStorage) NewList() runtime.Object { + return s.resourceInfo.NewListFunc() +} + +func (s *pluginsStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) { + return s.tableConverter.ConvertToTable(ctx, object, tableOptions) +} + +func (s *pluginsStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) { + return s.registry.GetDatasourceApiServers(ctx) +} diff --git a/pkg/registry/apis/query/query.go b/pkg/registry/apis/query/query.go new file mode 100644 index 00000000000..e5278141379 --- /dev/null +++ b/pkg/registry/apis/query/query.go @@ -0,0 +1,157 @@ +package query + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "golang.org/x/sync/errgroup" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/middleware/requestmeta" + "github.com/grafana/grafana/pkg/util/errutil/errhttp" + "github.com/grafana/grafana/pkg/web" +) + +func (b *QueryAPIBuilder) handleQuery(w http.ResponseWriter, r *http.Request) { + reqDTO := v0alpha1.GenericQueryRequest{} + if err := web.Bind(r, &reqDTO); err != nil { + errhttp.Write(r.Context(), err, w) + return + } + + parsed, err := parseQueryRequest(reqDTO) + if err != nil { + errhttp.Write(r.Context(), err, w) + return + } + + ctx := r.Context() + qdr, err := b.processRequest(ctx, parsed) + if err != nil { + errhttp.Write(r.Context(), err, w) + return + } + + statusCode := http.StatusOK + for _, res := range qdr.Responses { + if res.Error != nil { + statusCode = http.StatusBadRequest + if b.returnMultiStatus { + statusCode = http.StatusMultiStatus + } + } + } + if statusCode != http.StatusOK { + requestmeta.WithDownstreamStatusSource(ctx) + } + + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(qdr) + if err != nil { + errhttp.Write(r.Context(), err, w) + } +} + +// See: +// https://github.com/grafana/grafana/blob/v10.2.3/pkg/services/query/query.go#L88 +func (b *QueryAPIBuilder) processRequest(ctx context.Context, req parsedQueryRequest) (qdr *backend.QueryDataResponse, err error) { + switch len(req.Requests) { + case 0: + break // nothing to do + case 1: + qdr, err = b.handleQuerySingleDatasource(ctx, req.Requests[0]) + default: + qdr, err = b.executeConcurrentQueries(ctx, req.Requests) + } + + if len(req.Expressions) > 0 { + return b.handleExpressions(ctx, qdr, req.Expressions) + } + return qdr, err +} + +// Process a single request +// See: https://github.com/grafana/grafana/blob/v10.2.3/pkg/services/query/query.go#L242 +func (b *QueryAPIBuilder) handleQuerySingleDatasource(ctx context.Context, req groupedQueries) (*backend.QueryDataResponse, error) { + gv, err := b.registry.GetDatasourceGroupVersion(req.pluginId) + if err != nil { + return nil, err + } + return b.runner.ExecuteQueryData(ctx, gv, req.uid, req.query) +} + +// buildErrorResponses applies the provided error to each query response in the list. These queries should all belong to the same datasource. +func buildErrorResponse(err error, req groupedQueries) *backend.QueryDataResponse { + rsp := backend.NewQueryDataResponse() + for _, query := range req.query { + rsp.Responses[query.RefID] = backend.DataResponse{ + Error: err, + } + } + return rsp +} + +// executeConcurrentQueries executes queries to multiple datasources concurrently and returns the aggregate result. +func (b *QueryAPIBuilder) executeConcurrentQueries(ctx context.Context, requests []groupedQueries) (*backend.QueryDataResponse, error) { + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(b.concurrentQueryLimit) // prevent too many concurrent requests + rchan := make(chan *backend.QueryDataResponse, len(requests)) + + // Create panic recovery function for loop below + recoveryFn := func(req groupedQueries) { + if r := recover(); r != nil { + var err error + b.log.Error("query datasource panic", "error", r, "stack", log.Stack(1)) + if theErr, ok := r.(error); ok { + err = theErr + } else if theErrString, ok := r.(string); ok { + err = fmt.Errorf(theErrString) + } else { + err = fmt.Errorf("unexpected error - %s", b.userFacingDefaultError) + } + // Due to the panic, there is no valid response for any query for this datasource. Append an error for each one. + rchan <- buildErrorResponse(err, req) + } + } + + // Query each datasource concurrently + for idx := range requests { + req := requests[idx] + g.Go(func() error { + defer recoveryFn(req) + + dqr, err := b.handleQuerySingleDatasource(ctx, req) + if err == nil { + rchan <- dqr + } else { + rchan <- buildErrorResponse(err, req) + } + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + close(rchan) + + // Merge the results from each response + resp := backend.NewQueryDataResponse() + for result := range rchan { + for refId, dataResponse := range result.Responses { + resp.Responses[refId] = dataResponse + } + } + + return resp, nil +} + +// NOTE the upstream queries have already been executed +// https://github.com/grafana/grafana/blob/v10.2.3/pkg/services/query/query.go#L242 +func (b *QueryAPIBuilder) handleExpressions(ctx context.Context, qdr *backend.QueryDataResponse, expressions []v0alpha1.GenericDataQuery) (*backend.QueryDataResponse, error) { + return qdr, fmt.Errorf("expressions are not implemented yet") +} diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go new file mode 100644 index 00000000000..9b450c4904d --- /dev/null +++ b/pkg/registry/apis/query/register.go @@ -0,0 +1,260 @@ +package query + +import ( + "encoding/json" + "net/http" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/registry/generic" + "k8s.io/apiserver/pkg/registry/rest" + genericapiserver "k8s.io/apiserver/pkg/server" + common "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/spec3" + "k8s.io/kube-openapi/pkg/validation/spec" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/registry/apis/query/runner" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/featuremgmt" + grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" + "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" + "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" +) + +var _ grafanaapiserver.APIGroupBuilder = (*QueryAPIBuilder)(nil) + +type QueryAPIBuilder struct { + log log.Logger + concurrentQueryLimit int + userFacingDefaultError string + returnMultiStatus bool // from feature toggle + + runner v0alpha1.QueryRunner + registry v0alpha1.DataSourceApiServerRegistry +} + +func NewQueryAPIBuilder(features featuremgmt.FeatureToggles, + runner v0alpha1.QueryRunner, + registry v0alpha1.DataSourceApiServerRegistry, +) *QueryAPIBuilder { + return &QueryAPIBuilder{ + concurrentQueryLimit: 4, // from config? + log: log.New("query_apiserver"), + returnMultiStatus: features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryMultiStatus), + runner: runner, + registry: registry, + } +} + +func RegisterAPIService(features featuremgmt.FeatureToggles, + apiregistration grafanaapiserver.APIRegistrar, + dataSourcesService datasources.DataSourceService, + pluginStore pluginstore.Store, + accessControl accesscontrol.AccessControl, + pluginClient plugins.Client, + pCtxProvider *plugincontext.Provider, +) *QueryAPIBuilder { + if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { + return nil // skip registration unless opting into experimental apis + } + + builder := NewQueryAPIBuilder( + features, + runner.NewDirectQueryRunner(pluginClient, pCtxProvider), + runner.NewDirectRegistry(pluginStore, dataSourcesService), + ) + + // ONLY testdata... + if false { + builder = NewQueryAPIBuilder( + features, + runner.NewDummyTestRunner(), + runner.NewDummyRegistry(), + ) + } + + apiregistration.RegisterAPI(builder) + return builder +} + +func (b *QueryAPIBuilder) GetGroupVersion() schema.GroupVersion { + return v0alpha1.SchemeGroupVersion +} + +func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { + scheme.AddKnownTypes(gv, + &v0alpha1.DataSourceApiServer{}, + &v0alpha1.DataSourceApiServerList{}, + &v0alpha1.QueryDataResponse{}, + ) +} + +func (b *QueryAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { + addKnownTypes(scheme, v0alpha1.SchemeGroupVersion) + metav1.AddToGroupVersion(scheme, v0alpha1.SchemeGroupVersion) + return scheme.SetVersionPriority(v0alpha1.SchemeGroupVersion) +} + +func (b *QueryAPIBuilder) GetAPIGroupInfo( + scheme *runtime.Scheme, + codecs serializer.CodecFactory, // pointer? + optsGetter generic.RESTOptionsGetter, +) (*genericapiserver.APIGroupInfo, error) { + gv := v0alpha1.SchemeGroupVersion + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs) + + plugins := newPluginsStorage(b.registry) + + storage := map[string]rest.Storage{} + storage[plugins.resourceInfo.StoragePath()] = plugins + + apiGroupInfo.VersionedResourcesStorageMap[gv.Version] = storage + return &apiGroupInfo, nil +} + +func (b *QueryAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions { + return v0alpha1.GetOpenAPIDefinitions +} + +// Register additional routes with the server +func (b *QueryAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { + defs := v0alpha1.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) + querySchema := defs["github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryRequest"].Schema + responseSchema := defs["github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryDataResponse"].Schema + + var randomWalkQuery any + var randomWalkTable any + _ = json.Unmarshal([]byte(`{ + "queries": [ + { + "refId": "A", + "scenarioId": "random_walk", + "seriesCount": 1, + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "PD8C576611E62080A" + }, + "intervalMs": 60000, + "maxDataPoints": 20 + } + ], + "from": "1704893381544", + "to": "1704914981544" + }`), &randomWalkQuery) + + _ = json.Unmarshal([]byte(`{ + "queries": [ + { + "refId": "A", + "scenarioId": "random_walk_table", + "seriesCount": 1, + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "PD8C576611E62080A" + }, + "intervalMs": 60000, + "maxDataPoints": 20 + } + ], + "from": "1704893381544", + "to": "1704914981544" + }`), &randomWalkTable) + + return &grafanaapiserver.APIRoutes{ + Root: []grafanaapiserver.APIRouteHandler{}, + Namespace: []grafanaapiserver.APIRouteHandler{ + { + Path: "query", + Spec: &spec3.PathProps{ + Post: &spec3.Operation{ + OperationProps: spec3.OperationProps{ + Tags: []string{"query"}, + Description: "query across multiple datasources with expressions. This api matches the legacy /ds/query endpoint", + Parameters: []*spec3.Parameter{ + { + ParameterProps: spec3.ParameterProps{ + Name: "namespace", + Description: "object name and auth scope, such as for teams and projects", + In: "path", + Required: true, + Schema: spec.StringProperty(), + Example: "default", + }, + }, + }, + RequestBody: &spec3.RequestBody{ + RequestBodyProps: spec3.RequestBodyProps{ + Required: true, + Description: "the query array", + Content: map[string]*spec3.MediaType{ + "application/json": { + MediaTypeProps: spec3.MediaTypeProps{ + Schema: querySchema.WithExample(randomWalkQuery), + Examples: map[string]*spec3.Example{ + "random_walk": { + ExampleProps: spec3.ExampleProps{ + Summary: "random walk", + Value: randomWalkQuery, + }, + }, + "random_walk_table": { + ExampleProps: spec3.ExampleProps{ + Summary: "random walk (table)", + Value: randomWalkTable, + }, + }, + }, + }, + }, + }, + }, + }, + Responses: &spec3.Responses{ + ResponsesProps: spec3.ResponsesProps{ + StatusCodeResponses: map[int]*spec3.Response{ + http.StatusOK: { + ResponseProps: spec3.ResponseProps{ + Description: "Query results", + Content: map[string]*spec3.MediaType{ + "application/json": { + MediaTypeProps: spec3.MediaTypeProps{ + Schema: &responseSchema, + }, + }, + }, + }, + }, + http.StatusMultiStatus: { + ResponseProps: spec3.ResponseProps{ + Description: "Errors exist in the downstream results", + Content: map[string]*spec3.MediaType{ + "application/json": { + MediaTypeProps: spec3.MediaTypeProps{ + Schema: &responseSchema, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Handler: b.handleQuery, + }, + }, + } +} + +func (b *QueryAPIBuilder) GetAuthorizer() authorizer.Authorizer { + return nil // default is OK +} diff --git a/pkg/registry/apis/query/runner/direct.go b/pkg/registry/apis/query/runner/direct.go new file mode 100644 index 00000000000..e36a7e6188e --- /dev/null +++ b/pkg/registry/apis/query/runner/direct.go @@ -0,0 +1,179 @@ +package runner + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" + "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/legacydata" +) + +type directRunner struct { + pluginClient plugins.Client + pCtxProvider *plugincontext.Provider +} + +type directRegistry struct { + pluginsMu sync.Mutex + plugins *v0alpha1.DataSourceApiServerList + apis map[string]schema.GroupVersion + groupToPlugin map[string]string + pluginStore pluginstore.Store + + // called on demand + dataSourcesService datasources.DataSourceService +} + +var _ v0alpha1.QueryRunner = (*directRunner)(nil) +var _ v0alpha1.DataSourceApiServerRegistry = (*directRegistry)(nil) + +// NewDummyTestRunner creates a runner that only works with testdata +func NewDirectQueryRunner( + pluginClient plugins.Client, + pCtxProvider *plugincontext.Provider) v0alpha1.QueryRunner { + return &directRunner{ + pluginClient: pluginClient, + pCtxProvider: pCtxProvider, + } +} + +func NewDirectRegistry(pluginStore pluginstore.Store, + dataSourcesService datasources.DataSourceService, +) v0alpha1.DataSourceApiServerRegistry { + return &directRegistry{ + pluginStore: pluginStore, + dataSourcesService: dataSourcesService, + } +} + +// ExecuteQueryData implements QueryHelper. +func (d *directRunner) ExecuteQueryData(ctx context.Context, + // The k8s group for the datasource (pluginId) + datasource schema.GroupVersion, + + // The datasource name/uid + name string, + + // The raw backend query objects + query []v0alpha1.GenericDataQuery, +) (*backend.QueryDataResponse, error) { + queries, dsRef, err := legacydata.ToDataSourceQueries(v0alpha1.GenericQueryRequest{ + Queries: query, + }) + if err != nil { + return nil, err + } + if dsRef != nil && dsRef.UID != name { + return nil, fmt.Errorf("expected query body datasource and request to match") + } + + // NOTE: this depends on uid unique across datasources + settings, err := d.pCtxProvider.GetDataSourceInstanceSettings(ctx, name) + if err != nil { + return nil, err + } + + pCtx, err := d.pCtxProvider.PluginContextForDataSource(ctx, settings) + if err != nil { + return nil, err + } + + return d.pluginClient.QueryData(ctx, &backend.QueryDataRequest{ + PluginContext: pCtx, + Queries: queries, + }) +} + +// GetDatasourceAPI implements DataSourceRegistry. +func (d *directRegistry) GetDatasourceGroupVersion(pluginId string) (schema.GroupVersion, error) { + d.pluginsMu.Lock() + defer d.pluginsMu.Unlock() + + if d.plugins == nil { + err := d.updatePlugins() + if err != nil { + return schema.GroupVersion{}, err + } + } + + var err error + gv, ok := d.apis[pluginId] + if !ok { + err = fmt.Errorf("no API found for id: " + pluginId) + } + return gv, err +} + +// GetDatasourcePlugins no namespace? everything that is available +func (d *directRegistry) GetDatasourceApiServers(ctx context.Context) (*v0alpha1.DataSourceApiServerList, error) { + d.pluginsMu.Lock() + defer d.pluginsMu.Unlock() + + if d.plugins == nil { + err := d.updatePlugins() + if err != nil { + return nil, err + } + } + + return d.plugins, nil +} + +// This should be called when plugins change +func (d *directRegistry) updatePlugins() error { + groupToPlugin := map[string]string{} + apis := map[string]schema.GroupVersion{} + result := &v0alpha1.DataSourceApiServerList{ + ListMeta: metav1.ListMeta{ + ResourceVersion: fmt.Sprintf("%d", time.Now().UnixMilli()), + }, + } + + // TODO? only backend plugins + for _, dsp := range d.pluginStore.Plugins(context.Background(), plugins.TypeDataSource) { + ts := setting.BuildStamp * 1000 + if dsp.Info.Build.Time > 0 { + ts = dsp.Info.Build.Time + } + + group, err := plugins.GetDatasourceGroupNameFromPluginID(dsp.ID) + if err != nil { + return err + } + gv := schema.GroupVersion{Group: group, Version: "v0alpha1"} // default version + apis[dsp.ID] = gv + for _, alias := range dsp.AliasIDs { + apis[alias] = gv + } + groupToPlugin[group] = dsp.ID + + ds := v0alpha1.DataSourceApiServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsp.ID, + CreationTimestamp: metav1.NewTime(time.UnixMilli(ts)), + }, + Title: dsp.Name, + AliasIDs: dsp.AliasIDs, + GroupVersion: gv.String(), + Description: dsp.Info.Description, + } + result.Items = append(result.Items, ds) + } + + d.plugins = result + d.apis = apis + d.groupToPlugin = groupToPlugin + return nil +} diff --git a/pkg/registry/apis/query/runner/dummy.go b/pkg/registry/apis/query/runner/dummy.go new file mode 100644 index 00000000000..4937cb81241 --- /dev/null +++ b/pkg/registry/apis/query/runner/dummy.go @@ -0,0 +1,88 @@ +package runner + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + testdata "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource" + "github.com/grafana/grafana/pkg/tsdb/legacydata" +) + +type testdataDummy struct{} + +var _ v0alpha1.QueryRunner = (*testdataDummy)(nil) +var _ v0alpha1.DataSourceApiServerRegistry = (*testdataDummy)(nil) + +// NewDummyTestRunner creates a runner that only works with testdata +func NewDummyTestRunner() v0alpha1.QueryRunner { + return &testdataDummy{} +} + +func NewDummyRegistry() v0alpha1.DataSourceApiServerRegistry { + return &testdataDummy{} +} + +// ExecuteQueryData implements QueryHelper. +func (d *testdataDummy) ExecuteQueryData(ctx context.Context, + // The k8s group for the datasource (pluginId) + datasource schema.GroupVersion, + + // The datasource name/uid + name string, + + // The raw backend query objects + query []v0alpha1.GenericDataQuery, +) (*backend.QueryDataResponse, error) { + if datasource.Group != "testdata.datasource.grafana.app" { + return nil, fmt.Errorf("expecting testdata requests") + } + + queries, _, err := legacydata.ToDataSourceQueries(v0alpha1.GenericQueryRequest{ + Queries: query, + }) + if err != nil { + return nil, err + } + + return testdata.ProvideService().QueryData(ctx, &backend.QueryDataRequest{ + Queries: queries, + }) +} + +// GetDatasourceAPI implements DataSourceRegistry. +func (*testdataDummy) GetDatasourceGroupVersion(pluginId string) (schema.GroupVersion, error) { + if pluginId == "testdata" || pluginId == "grafana-testdata-datasource" { + return schema.GroupVersion{ + Group: "testdata.datasource.grafana.app", + Version: "v0alpha1", + }, nil + } + return schema.GroupVersion{}, fmt.Errorf("unsupported plugin (only testdata for now)") +} + +// GetDatasourcePlugins implements QueryHelper. +func (d *testdataDummy) GetDatasourceApiServers(ctx context.Context) (*v0alpha1.DataSourceApiServerList, error) { + return &v0alpha1.DataSourceApiServerList{ + ListMeta: metav1.ListMeta{ + ResourceVersion: fmt.Sprintf("%d", time.Now().UnixMilli()), + }, + Items: []v0alpha1.DataSourceApiServer{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "grafana-testdata-datasource", + CreationTimestamp: metav1.Now(), + }, + Title: "Test Data", + GroupVersion: "testdata.datasource.grafana.app/v0alpha1", + AliasIDs: []string{"testdata"}, + }, + }, + }, nil +} diff --git a/pkg/registry/apis/wireset.go b/pkg/registry/apis/wireset.go index 3497faba067..c5da75bf909 100644 --- a/pkg/registry/apis/wireset.go +++ b/pkg/registry/apis/wireset.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/featuretoggle" "github.com/grafana/grafana/pkg/registry/apis/folders" "github.com/grafana/grafana/pkg/registry/apis/playlist" + "github.com/grafana/grafana/pkg/registry/apis/query" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" ) @@ -27,4 +28,5 @@ var WireSet = wire.NewSet( featuretoggle.RegisterAPIService, datasource.RegisterAPIService, folders.RegisterAPIService, + query.RegisterAPIService, ) diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index bfab3b7a3eb..f790618239a 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -949,6 +949,15 @@ var ( RequiresRestart: true, // changes the API routing Created: time.Date(2023, time.December, 4, 12, 0, 0, 0, time.UTC), }, + { + Name: "kubernetesQueryServiceRewrite", + Description: "Rewrite requests targeting /ds/query to the query service", + Stage: FeatureStageExperimental, + Owner: grafanaAppPlatformSquad, + RequiresRestart: true, // changes the API routing + RequiresDevMode: true, + Created: time.Date(2024, time.January, 28, 12, 0, 0, 0, time.UTC), + }, { Name: "cloudWatchBatchQueries", Description: "Runs CloudWatch metrics queries as separate batches", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 197563171e8..ac27e659c64 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -111,6 +111,7 @@ formatString,preview,@grafana/grafana-bi-squad,2023-10-13,false,false,true transformationsVariableSupport,preview,@grafana/grafana-bi-squad,2023-10-04,false,false,true kubernetesPlaylists,experimental,@grafana/grafana-app-platform-squad,2023-11-08,false,true,false kubernetesSnapshots,experimental,@grafana/grafana-app-platform-squad,2023-12-04,false,true,false +kubernetesQueryServiceRewrite,experimental,@grafana/grafana-app-platform-squad,2024-01-28,true,true,false cloudWatchBatchQueries,preview,@grafana/aws-datasources,2023-10-20,false,false,false recoveryThreshold,GA,@grafana/alerting-squad,2023-10-10,false,true,false lokiStructuredMetadata,experimental,@grafana/observability-logs,2023-11-16,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 2c0e677c0c3..42b161e6701 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -455,6 +455,10 @@ const ( // Use the kubernetes API in the frontend to support playlists FlagKubernetesSnapshots = "kubernetesSnapshots" + // FlagKubernetesQueryServiceRewrite + // Rewrite requests targeting /ds/query to the query service + FlagKubernetesQueryServiceRewrite = "kubernetesQueryServiceRewrite" + // FlagCloudWatchBatchQueries // Runs CloudWatch metrics queries as separate batches FlagCloudWatchBatchQueries = "cloudWatchBatchQueries" diff --git a/pkg/services/grafana-apiserver/service.go b/pkg/services/grafana-apiserver/service.go index 30519427505..84447d9db2f 100644 --- a/pkg/services/grafana-apiserver/service.go +++ b/pkg/services/grafana-apiserver/service.go @@ -103,6 +103,9 @@ type DirectRestConfigProvider interface { // logged logged in user as the current request context. This is useful when // creating clients that map legacy API handlers to k8s backed services GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Config + + // This can be used to rewrite incoming requests to path now supported under /apis + DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) } type service struct { @@ -377,6 +380,10 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co } } +func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) { + s.handler.ServeHTTP(w, r) +} + func (s *service) running(ctx context.Context) error { // skip waiting for the server in prod mode if !s.config.devMode { diff --git a/pkg/tests/apis/datasource/testdata_test.go b/pkg/tests/apis/datasource/testdata_test.go index c21b1bdf309..b0449903fe3 100644 --- a/pkg/tests/apis/datasource/testdata_test.go +++ b/pkg/tests/apis/datasource/testdata_test.go @@ -107,7 +107,7 @@ func TestTestDatasource(t *testing.T) { }) t.Run("Call subresources", func(t *testing.T) { - client := helper.Org1.Admin.Client.Resource(schema.GroupVersionResource{ + client := helper.Org1.Admin.ResourceClient(t, schema.GroupVersionResource{ Group: "testdata.datasource.grafana.app", Version: "v0alpha1", Resource: "connections", diff --git a/pkg/tests/apis/example/example_test.go b/pkg/tests/apis/example/example_test.go index 6f3f6f423f6..b84db254f2f 100644 --- a/pkg/tests/apis/example/example_test.go +++ b/pkg/tests/apis/example/example_test.go @@ -29,7 +29,7 @@ func TestExampleApp(t *testing.T) { t.Run("Check runtime info resource", func(t *testing.T) { // Resource is not namespaced! - client := helper.Org1.Admin.Client.Resource(schema.GroupVersionResource{ + client := helper.Org1.Admin.ResourceClient(t, schema.GroupVersionResource{ Group: "example.grafana.app", Version: "v0alpha1", Resource: "runtime", @@ -139,7 +139,7 @@ func TestExampleApp(t *testing.T) { }) t.Run("Check dummy with subresource", func(t *testing.T) { - client := helper.Org1.Viewer.Client.Resource(schema.GroupVersionResource{ + client := helper.Org1.Viewer.ResourceClient(t, schema.GroupVersionResource{ Group: "example.grafana.app", Version: "v0alpha1", Resource: "dummy", diff --git a/pkg/tests/apis/helper.go b/pkg/tests/apis/helper.go index ee7e2c24aff..3d1e233174d 100644 --- a/pkg/tests/apis/helper.go +++ b/pkg/tests/apis/helper.go @@ -98,10 +98,13 @@ func (c *K8sTestHelper) GetResourceClient(args ResourceClientArgs) *K8sResourceC args.Namespace = c.namespacer(args.User.Identity.GetOrgID()) } + client, err := dynamic.NewForConfig(args.User.NewRestConfig()) + require.NoError(c.t, err) + return &K8sResourceClient{ t: c.t, Args: args, - Resource: args.User.Client.Resource(args.GVR).Namespace(args.Namespace), + Resource: client.Resource(args.GVR).Namespace(args.Namespace), } } @@ -163,8 +166,31 @@ type OrgUsers struct { type User struct { Identity identity.Requester - Client *dynamic.DynamicClient password string + baseURL string +} + +func (c *User) NewRestConfig() *rest.Config { + return &rest.Config{ + Host: c.baseURL, + Username: c.Identity.GetLogin(), + Password: c.password, + } +} + +func (c *User) ResourceClient(t *testing.T, gvr schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { + client, err := dynamic.NewForConfig(c.NewRestConfig()) + require.NoError(t, err) + return client.Resource(gvr) +} + +func (c *User) RESTClient(t *testing.T, gv *schema.GroupVersion) *rest.RESTClient { + cfg := dynamic.ConfigFor(c.NewRestConfig()) // adds negotiated serializers! + cfg.GroupVersion = gv + cfg.APIPath = "apis" // the plural + client, err := rest.RESTClientFor(cfg) + require.NoError(t, err) + return client } type RequestParams struct { @@ -380,19 +406,10 @@ func (c K8sTestHelper) createTestUsers(orgName string) OrgUsers { require.Equal(c.t, orgId, s.OrgID) require.Equal(c.t, role, s.OrgRole) // make sure the role was set properly - config := &rest.Config{ - Host: baseUrl, - Username: s.Login, - Password: key, - } - - client, err := dynamic.NewForConfig(config) - require.NoError(c.t, err) - return User{ Identity: s, - Client: client, password: key, + baseURL: baseUrl, } } return OrgUsers{ diff --git a/pkg/tests/apis/query/query_test.go b/pkg/tests/apis/query/query_test.go new file mode 100644 index 00000000000..8a70de87986 --- /dev/null +++ b/pkg/tests/apis/query/query_test.go @@ -0,0 +1,98 @@ +package dashboards + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + + query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/tests/apis" + "github.com/grafana/grafana/pkg/tests/testinfra" +) + +func TestSimpleQuery(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + AppModeProduction: false, // dev mode required for datasource connections + EnableFeatureToggles: []string{ + featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + }, + }) + + // Create a single datasource + ds := helper.CreateDS(&datasources.AddDataSourceCommand{ + Name: "test", + Type: datasources.DS_TESTDATA, + UID: "test", + OrgID: int64(1), + }) + require.Equal(t, "test", ds.UID) + + t.Run("Call query", func(t *testing.T) { + client := helper.Org1.Admin.RESTClient(t, &schema.GroupVersion{ + Group: "query.grafana.app", + Version: "v0alpha1", + }) + + q := query.GenericDataQuery{ + Datasource: &query.DataSourceRef{ + Type: "grafana-testdata-datasource", + UID: ds.UID, + }, + } + q.AdditionalProperties()["csvContent"] = "a,b,c\n1,hello,true" + q.AdditionalProperties()["scenarioId"] = "csv_content" + body, err := json.Marshal(&query.GenericQueryRequest{Queries: []query.GenericDataQuery{q}}) + require.NoError(t, err) + + result := client.Post(). + Namespace("default"). + Suffix("query"). + SetHeader("Content-type", "application/json"). + Body(body). + Do(context.Background()) + + require.NoError(t, result.Error()) + + body, err = result.Raw() + require.NoError(t, err) + fmt.Printf("OUT: %s", string(body)) + + rsp := &backend.QueryDataResponse{} + err = json.Unmarshal(body, rsp) + require.NoError(t, err) + require.Equal(t, 1, len(rsp.Responses)) + + frame := rsp.Responses["A"].Frames[0] + disp, err := frame.StringTable(100, 10) + require.NoError(t, err) + fmt.Printf("%s\n", disp) + + type expect struct { + idx int + name string + val any + } + for _, check := range []expect{ + {0, "a", int64(1)}, + {1, "b", "hello"}, + {2, "c", true}, + } { + field := frame.Fields[check.idx] + require.Equal(t, check.name, field.Name) + + v, _ := field.ConcreteAt(0) + require.Equal(t, check.val, v) + } + }) +} diff --git a/pkg/tsdb/legacydata/conversions.go b/pkg/tsdb/legacydata/conversions.go new file mode 100644 index 00000000000..37eeaffd53f --- /dev/null +++ b/pkg/tsdb/legacydata/conversions.go @@ -0,0 +1,85 @@ +package legacydata + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + + "github.com/grafana/grafana/pkg/apis/query/v0alpha1" +) + +// ToDataSourceQueries returns queries that should be sent to a single datasource +// This will throw an error if the queries reference multiple instances +func ToDataSourceQueries(req v0alpha1.GenericQueryRequest) ([]backend.DataQuery, *v0alpha1.DataSourceRef, error) { + var dsRef *v0alpha1.DataSourceRef + var tr *backend.TimeRange + if req.From != "" { + val := NewDataTimeRange(req.From, req.To) + tr = &backend.TimeRange{ + From: val.GetFromAsTimeUTC(), + To: val.GetToAsTimeUTC(), + } + } + + queries := []backend.DataQuery{} + if len(req.Queries) > 0 { + dsRef := req.Queries[0].Datasource + for _, generic := range req.Queries { + if generic.Datasource != nil && dsRef != nil { + if dsRef.Type != generic.Datasource.Type { + return queries, dsRef, fmt.Errorf("expect same datasource types") + } + if dsRef.UID != generic.Datasource.UID { + return queries, dsRef, fmt.Errorf("expect same datasource UID") + } + } + q, err := toBackendDataQuery(generic, tr) + if err != nil { + return queries, dsRef, err + } + queries = append(queries, q) + } + return queries, dsRef, nil + } + return queries, dsRef, nil +} + +// Converts a generic query to a backend one +func toBackendDataQuery(q v0alpha1.GenericDataQuery, defaultTimeRange *backend.TimeRange) (backend.DataQuery, error) { + var err error + bq := backend.DataQuery{ + RefID: q.RefID, + QueryType: q.QueryType, + MaxDataPoints: q.MaxDataPoints, + } + + // Set an explicit time range for the query + if q.TimeRange != nil { + tr := NewDataTimeRange(q.TimeRange.From, q.TimeRange.To) + bq.TimeRange = backend.TimeRange{ + From: tr.GetFromAsTimeUTC(), + To: tr.GetToAsTimeUTC(), + } + } else if defaultTimeRange != nil { + bq.TimeRange = *defaultTimeRange + } + + bq.JSON, err = json.Marshal(q) + if err != nil { + return bq, err + } + if bq.RefID == "" { + bq.RefID = "A" + } + if bq.MaxDataPoints == 0 { + bq.MaxDataPoints = 100 + } + if q.IntervalMS > 0 { + bq.Interval = time.Millisecond * time.Duration(q.IntervalMS) + } else { + bq.Interval = time.Second + } + return bq, nil +} From 0ce1ccd6f969485bbb896fb4b768faf5c5a933e6 Mon Sep 17 00:00:00 2001 From: Matthew Jacobson Date: Wed, 31 Jan 2024 14:05:30 -0500 Subject: [PATCH 24/84] Alerting: Fix inconsistent AM raw config when applied via sync vs API (#81655) AM config applied via API would use the PostableUserConfig as the AM raw config and also the hash used to decide when the AM config has changed. However, when applied via the periodic sync the PostableApiAlertingConfig would be used instead. This leads to two issues: - Inconsistent hash comparisons when modifying the AM causing redundant applies. - GetStatus assumed the raw config was PostableUserConfig causing the endpoint to return correctly after a new config is applied via API and then nothing once the periodic sync runs. Note: Technically, the upstream GrafanaAlertamanger GetStatus shouldn't be returning PostableUserConfig or PostableApiAlertingConfig, but instead GettableStatus. However, this issue required changes elsewhere and is out of scope. --- pkg/services/ngalert/notifier/alertmanager.go | 23 ++++++++----------- pkg/services/ngalert/notifier/status.go | 8 +++---- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index eb06c23ae7f..980a66905cf 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -180,7 +180,7 @@ func (am *alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error { } err = am.Store.SaveAlertmanagerConfigurationWithCallback(ctx, cmd, func() error { - _, err := am.applyConfig(cfg, []byte(am.Settings.UnifiedAlerting.DefaultConfiguration)) + _, err := am.applyConfig(cfg) return err }) if err != nil { @@ -210,7 +210,7 @@ func (am *alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P } err = am.Store.SaveAlertmanagerConfigurationWithCallback(ctx, cmd, func() error { - _, err := am.applyConfig(cfg, rawConfig) + _, err := am.applyConfig(cfg) return err }) if err != nil { @@ -232,7 +232,7 @@ func (am *alertmanager) ApplyConfig(ctx context.Context, dbCfg *ngmodels.AlertCo var outerErr error am.Base.WithLock(func() { - if err := am.applyAndMarkConfig(ctx, dbCfg.ConfigurationHash, cfg, nil); err != nil { + if err := am.applyAndMarkConfig(ctx, dbCfg.ConfigurationHash, cfg); err != nil { outerErr = fmt.Errorf("unable to apply configuration: %w", err) return } @@ -286,16 +286,13 @@ func (am *alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu // applyConfig applies a new configuration by re-initializing all components using the configuration provided. // It returns a boolean indicating whether the user config was changed and an error. // It is not safe to call concurrently. -func (am *alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig []byte) (bool, error) { +func (am *alertmanager) applyConfig(cfg *apimodels.PostableUserConfig) (bool, error) { // First, let's make sure this config is not already loaded var amConfigChanged bool - if rawConfig == nil { - enc, err := json.Marshal(cfg.AlertmanagerConfig) - if err != nil { - // In theory, this should never happen. - return false, err - } - rawConfig = enc + rawConfig, err := json.Marshal(cfg.AlertmanagerConfig) + if err != nil { + // In theory, this should never happen. + return false, err } if am.Base.ConfigHash() != md5.Sum(rawConfig) { @@ -335,8 +332,8 @@ func (am *alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig } // applyAndMarkConfig applies a configuration and marks it as applied if no errors occur. -func (am *alertmanager) applyAndMarkConfig(ctx context.Context, hash string, cfg *apimodels.PostableUserConfig, rawConfig []byte) error { - configChanged, err := am.applyConfig(cfg, rawConfig) +func (am *alertmanager) applyAndMarkConfig(ctx context.Context, hash string, cfg *apimodels.PostableUserConfig) error { + configChanged, err := am.applyConfig(cfg) if err != nil { return err } diff --git a/pkg/services/ngalert/notifier/status.go b/pkg/services/ngalert/notifier/status.go index bae821f6d40..e93dbffd010 100644 --- a/pkg/services/ngalert/notifier/status.go +++ b/pkg/services/ngalert/notifier/status.go @@ -8,15 +8,15 @@ import ( // TODO: We no longer do apimodels at this layer, move it to the API. func (am *alertmanager) GetStatus() apimodels.GettableStatus { - config := &apimodels.PostableUserConfig{} - status := am.Base.GetStatus() + config := &apimodels.PostableApiAlertingConfig{} + status := am.Base.GetStatus() // TODO: This should return a GettableStatus, for now it returns PostableApiAlertingConfig. if status == nil { - return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) + return *apimodels.NewGettableStatus(config) } if err := json.Unmarshal(status, config); err != nil { am.logger.Error("Unable to unmarshall alertmanager config", "Err", err) } - return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) + return *apimodels.NewGettableStatus(config) } From 11997a6d3577793320b893e2071fb9535b871eb0 Mon Sep 17 00:00:00 2001 From: Dai Nguyen <88277570+ej25a@users.noreply.github.com> Date: Wed, 31 Jan 2024 17:03:08 -0600 Subject: [PATCH 25/84] Docs: Grafana.com skip_org_role_sync update (#80770) * Update Grafana.com org sync index.md Included additional information regarding logging in with Grafana.com credentials that it will override what is defined within the Grafana instance. * Update docs/sources/setup-grafana/configure-security/configure-authentication/grafana-com/index.md Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com> --------- Co-authored-by: Eve Meelan <81647476+Eve832@users.noreply.github.com> Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com> --- .../configure-authentication/grafana-com/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup-grafana/configure-security/configure-authentication/grafana-com/index.md b/docs/sources/setup-grafana/configure-security/configure-authentication/grafana-com/index.md index ffed7b045e8..19a15256d91 100644 --- a/docs/sources/setup-grafana/configure-security/configure-authentication/grafana-com/index.md +++ b/docs/sources/setup-grafana/configure-security/configure-authentication/grafana-com/index.md @@ -50,7 +50,7 @@ auto_login = true ## Skip organization role sync -To prevent the sync of org roles from Grafana.com, set `skip_org_role_sync` to `true`. This is useful if you want to manage the organization roles for your users from within Grafana. +If a user signs in with their Grafana.com credentials, their assigned org role overrides the role defined in the Grafana instance. To prevent Grafana.com roles from synchronizing, set `skip_org_role_sync` to `true`. This is useful if you want to manage the organization roles for your users from within Grafana. ```ini [auth.grafana_com] From c9bdf69a46ea7729cdebd0c01a93c39d84fedc05 Mon Sep 17 00:00:00 2001 From: ajwerner Date: Wed, 31 Jan 2024 18:08:40 -0500 Subject: [PATCH 26/84] Stat: Fix data links that refer to fields (#80693) --- .../grafana-data/src/field/fieldDisplay.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/grafana-data/src/field/fieldDisplay.ts b/packages/grafana-data/src/field/fieldDisplay.ts index 9833f10e2a7..ff33188eff3 100644 --- a/packages/grafana-data/src/field/fieldDisplay.ts +++ b/packages/grafana-data/src/field/fieldDisplay.ts @@ -204,6 +204,25 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi } } + // If there is only one row in the data frame, then set the + // valueRowIndex to that one row. This allows the data macros in + // things like links to access other fields from the data frame. + // + // If there were more rows, it still may be sane to set the row + // index, but it may be confusing; the calculation may have + // selected a value from a different row or it may have aggregated + // the values from multiple rows, so to make just the first row + // available would be arbitrary. For now, the users will have to + // ensure that the data frame has just one row if they want data + // link referencing other fields to work. + // + // TODO: A more complete solution here would be to allow the + // calculation to report a relevant row and use that value. For + // example, a common calculation is 'lastNotNull'. It'd be nifty to + // know which row the display value corresponds to in that case if + // there were potentially many + const valueRowIndex = dataFrame.length === 1 ? 0 : undefined; + values.push({ name: calc, field: config, @@ -215,6 +234,7 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi ? () => fieldLinksSupplier({ calculatedValue: displayValue, + valueRowIndex, }) : () => [], hasLinks: hasLinks(field), From 177fa1b947ee06f1143833b5406055ccf258615b Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 1 Feb 2024 00:30:21 +0100 Subject: [PATCH 27/84] InfluxDB: Fix template variable interpolation (#80971) * use regex as templateSrv replace format * use regex as templateSrv replace format for raw queries * import path fix * don't use regex formatter * tag value escape * tag value escape with wrappers * polished interpolation logic * update unit tests * comments and more place to update * unit test update * fix escaping * handle the string and array of string type separately * update variable type --- pkg/tsdb/influxdb/models/query.go | 70 +++++++++-- pkg/tsdb/influxdb/models/query_test.go | 48 +++++++ .../datasource/influxdb/datasource.test.ts | 105 +++++++++++++--- .../plugins/datasource/influxdb/datasource.ts | 117 ++++++++++-------- .../influxdb/datasource_backend_mode.test.ts | 22 ++-- .../app/plugins/datasource/influxdb/mocks.ts | 4 +- 6 files changed, 284 insertions(+), 82 deletions(-) diff --git a/pkg/tsdb/influxdb/models/query.go b/pkg/tsdb/influxdb/models/query.go index e8e6837aae7..64f3331a37f 100644 --- a/pkg/tsdb/influxdb/models/query.go +++ b/pkg/tsdb/influxdb/models/query.go @@ -13,8 +13,11 @@ import ( ) var ( - regexpOperatorPattern = regexp.MustCompile(`^\/.*\/$`) - regexpMeasurementPattern = regexp.MustCompile(`^\/.*\/$`) + regexpOperatorPattern = regexp.MustCompile(`^\/.*\/$`) + regexpMeasurementPattern = regexp.MustCompile(`^\/.*\/$`) + regexMatcherSimple = regexp.MustCompile(`^/(.*)/$`) + regexMatcherWithStartEndPattern = regexp.MustCompile(`^/\^(.*)\$/$`) + mustEscapeCharsMatcher = regexp.MustCompile(`[\\^$*+?.()|[\]{}\/]`) ) func (query *Query) Build(queryContext *backend.QueryDataRequest) (string, error) { @@ -103,20 +106,20 @@ func (query *Query) renderTags() []string { textValue = fmt.Sprintf("'%s'", strings.ReplaceAll(tag.Value, `\`, `\\`)) } - return textValue, operator + return removeRegexWrappers(textValue, `'`), operator } // quote value unless regex or number var textValue string switch tag.Operator { - case "=~", "!~": - textValue = tag.Value + case "=~", "!~", "": + textValue = escape(tag.Value) case "<", ">", ">=", "<=": - textValue = tag.Value + textValue = removeRegexWrappers(tag.Value, `'`) case "Is", "Is Not": textValue, tag.Operator = isOperatorTypeHandler(tag) default: - textValue = fmt.Sprintf("'%s'", strings.ReplaceAll(tag.Value, `\`, `\\`)) + textValue = fmt.Sprintf("'%s'", strings.ReplaceAll(removeRegexWrappers(tag.Value, ""), `\`, `\\`)) } escapedKey := fmt.Sprintf(`"%s"`, tag.Key) @@ -244,3 +247,56 @@ func epochMStoInfluxTime(tr *backend.TimeRange) (string, string) { return fmt.Sprintf("%dms", from), fmt.Sprintf("%dms", to) } + +func removeRegexWrappers(wrappedValue string, wrapper string) string { + value := wrappedValue + // get the value only in between /^...$/ + matches := regexMatcherWithStartEndPattern.FindStringSubmatch(wrappedValue) + if len(matches) > 1 { + // full match. the value is like /^value$/ + value = wrapper + matches[1] + wrapper + } + + return value +} + +func escape(unescapedValue string) string { + pipe := `|` + beginning := `/^` + ending := `$/` + value := unescapedValue + substitute := `\$0` + fullMatch := false + + // get the value only in between /^...$/ + matches := regexMatcherWithStartEndPattern.FindStringSubmatch(unescapedValue) + if len(matches) > 1 { + // full match. the value is like /^value$/ + value = matches[1] + fullMatch = true + } + + if !fullMatch { + // get the value only in between /.../ + matches = regexMatcherSimple.FindStringSubmatch(unescapedValue) + if len(matches) > 1 { + value = matches[1] + beginning = `/` + ending = `/` + } + } + + // split them with pipe | + parts := strings.Split(value, pipe) + for i, v := range parts { + // escape each item + parts[i] = mustEscapeCharsMatcher.ReplaceAllString(v, substitute) + } + + // stitch them to each other + escaped := make([]byte, 0, 64) + escaped = append(escaped, beginning...) + escaped = append(escaped, strings.Join(parts, pipe)...) + escaped = append(escaped, ending...) + return string(escaped) +} diff --git a/pkg/tsdb/influxdb/models/query_test.go b/pkg/tsdb/influxdb/models/query_test.go index c590e829d16..35c8bbd9f22 100644 --- a/pkg/tsdb/influxdb/models/query_test.go +++ b/pkg/tsdb/influxdb/models/query_test.go @@ -303,5 +303,53 @@ func TestInfluxdbQueryBuilder(t *testing.T) { require.Equal(t, query.renderMeasurement(), ` FROM "policy"./apa/`) }) + + t.Run("can render regexp tags", func(t *testing.T) { + query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/etc/hosts|/etc/hostname`, Key: "key"}}} + + require.Equal(t, `"key" =~ /^\/etc\/hosts|\/etc\/hostname$/`, strings.Join(query.renderTags(), "")) + }) + + t.Run("can render regexp tags 2", func(t *testing.T) { + query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/^/etc/hosts$/`, Key: "key"}}} + + require.Equal(t, `"key" =~ /^\/etc\/hosts$/`, strings.Join(query.renderTags(), "")) + }) + + t.Run("can render regexp tags 3", func(t *testing.T) { + query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/etc/hosts`, Key: "key"}}} + + require.Equal(t, `"key" =~ /^\/etc\/hosts$/`, strings.Join(query.renderTags(), "")) + }) + + t.Run("can render regexp tags with dots in values", func(t *testing.T) { + query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/etc/resolv.conf`, Key: "key"}}} + + require.Equal(t, `"key" =~ /^\/etc\/resolv\.conf$/`, strings.Join(query.renderTags(), "")) + }) + + t.Run("can render single quoted tag value when regexed value has been sent", func(t *testing.T) { + query := &Query{Tags: []*Tag{{Operator: ">", Value: `/^12.2$/`, Key: "key"}}} + + require.Equal(t, `"key" > '12.2'`, strings.Join(query.renderTags(), "")) + }) + }) +} + +func TestRemoveRegexWrappers(t *testing.T) { + t.Run("remove regex wrappers", func(t *testing.T) { + wrappedText := `/^someValue$/` + expected := `'someValue'` + result := removeRegexWrappers(wrappedText, `'`) + + require.Equal(t, expected, result) + }) + + t.Run("return same value if the value is not wrapped by regex wrappers", func(t *testing.T) { + wrappedText := `someValue` + expected := `someValue` + result := removeRegexWrappers(wrappedText, "") + + require.Equal(t, expected, result) }) } diff --git a/public/app/plugins/datasource/influxdb/datasource.test.ts b/public/app/plugins/datasource/influxdb/datasource.test.ts index 405f6b41be5..a5eb3beabab 100644 --- a/public/app/plugins/datasource/influxdb/datasource.test.ts +++ b/public/app/plugins/datasource/influxdb/datasource.test.ts @@ -5,9 +5,10 @@ import { BackendSrvRequest } from '@grafana/runtime/'; import config from 'app/core/config'; import { TemplateSrv } from '../../../features/templating/template_srv'; +import { queryBuilder } from '../../../features/variables/shared/testing/builders'; import { BROWSER_MODE_DISABLED_MESSAGE } from './constants'; -import InfluxDatasource, { influxSpecialRegexEscape } from './datasource'; +import InfluxDatasource from './datasource'; import { getMockDSInstanceSettings, getMockInfluxDS, @@ -258,6 +259,7 @@ describe('InfluxDataSource Frontend Mode', () => { const text2 = 'interpolationText2'; const textWithoutFormatRegex = 'interpolationText,interpolationText2'; const textWithFormatRegex = 'interpolationText,interpolationText2'; + const justText = 'interpolationText'; const variableMap: Record = { $interpolationVar: text, $interpolationVar2: text2, @@ -287,14 +289,14 @@ describe('InfluxDataSource Frontend Mode', () => { function influxChecks(query: InfluxQuery) { expect(templateSrv.replace).toBeCalledTimes(12); expect(query.alias).toBe(text); - expect(query.measurement).toBe(textWithFormatRegex); - expect(query.policy).toBe(textWithFormatRegex); - expect(query.limit).toBe(textWithFormatRegex); - expect(query.slimit).toBe(textWithFormatRegex); + expect(query.measurement).toBe(justText); + expect(query.policy).toBe(justText); + expect(query.limit).toBe(justText); + expect(query.slimit).toBe(justText); expect(query.tz).toBe(text); expect(query.tags![0].value).toBe(textWithFormatRegex); - expect(query.groupBy![0].params![0]).toBe(textWithFormatRegex); - expect(query.select![0][0].params![0]).toBe(textWithFormatRegex); + expect(query.groupBy![0].params![0]).toBe(justText); + expect(query.select![0][0].params![0]).toBe(justText); expect(query.adhocFilters?.[0].key).toBe(adhocFilters[0].key); } @@ -342,7 +344,12 @@ describe('InfluxDataSource Frontend Mode', () => { }); describe('variable interpolation with chained variables with frontend mode', () => { - const mockTemplateService = new TemplateSrv(); + const variablesMock = [queryBuilder().withId('var1').withName('var1').withCurrent('var1').build()]; + const mockTemplateService = new TemplateSrv({ + getVariables: () => variablesMock, + getVariableWithName: (name: string) => variablesMock.filter((v) => v.name === name)[0], + getFilteredVariables: jest.fn(), + }); mockTemplateService.getAdhocFilters = jest.fn((_: string) => []); let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService); const fetchMockImpl = () => @@ -410,18 +417,82 @@ describe('InfluxDataSource Frontend Mode', () => { }); }); - describe('influxSpecialRegexEscape', () => { - it('should escape the dot properly', () => { - const value = 'value.with-dot'; - const expectation = `value\.with-dot`; - const result = influxSpecialRegexEscape(value); + describe('interpolateQueryExpr', () => { + let ds = getMockInfluxDS(getMockDSInstanceSettings(), new TemplateSrv()); + it('should return the value as it is', () => { + const value = 'normalValue'; + const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build(); + const result = ds.interpolateQueryExpr(value, variableMock, 'my query $tempVar'); + const expectation = 'normalValue'; expect(result).toBe(expectation); }); - it('should escape the url properly', () => { - const value = 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/jolokia'; - const expectation = `https:\/\/aaaa-aa-aaa\.bbb\.ccc\.ddd:8443\/jolokia`; - const result = influxSpecialRegexEscape(value); + it('should return the escaped value if the value wrapped in regex', () => { + const value = '/special/path'; + const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build(); + const result = ds.interpolateQueryExpr(value, variableMock, 'select that where path = /$tempVar/'); + const expectation = `\\/special\\/path`; + expect(result).toBe(expectation); + }); + + it('should return the escaped value if the value wrapped in regex 2', () => { + const value = '/special/path'; + const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build(); + const result = ds.interpolateQueryExpr(value, variableMock, 'select that where path = /^$tempVar$/'); + const expectation = `\\/special\\/path`; + expect(result).toBe(expectation); + }); + + it('should **not** return the escaped value if the value **is not** wrapped in regex', () => { + const value = '/special/path'; + const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build(); + const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = '$tempVar'`); + const expectation = `/special/path`; + expect(result).toBe(expectation); + }); + + it('should **not** return the escaped value if the value **is not** wrapped in regex 2', () => { + const value = '12.2'; + const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build(); + const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = '$tempVar'`); + const expectation = `12.2`; + expect(result).toBe(expectation); + }); + + it('should escape the value **always** if the variable is a multi-value variable', () => { + const value = [`/special/path`, `/some/other/path`]; + const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti().build(); + const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = '$tempVar'`); + const expectation = `\\/special\\/path|\\/some\\/other\\/path`; + expect(result).toBe(expectation); + }); + + it('should escape and join with the pipe even the variable is not multi-value', () => { + const variableMock = queryBuilder() + .withId('tempVar') + .withName('tempVar') + .withCurrent('All', '$__all') + .withMulti(false) + .withAllValue('') + .withIncludeAll() + .withOptions( + { + text: 'All', + value: '$__all', + }, + { + text: `/special/path`, + value: `/special/path`, + }, + { + text: `/some/other/path`, + value: `/some/other/path`, + } + ) + .build(); + const value = [`/special/path`, `/some/other/path`]; + const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = /$tempVar/`); + const expectation = `\\/special\\/path|\\/some\\/other\\/path`; expect(result).toBe(expectation); }); }); diff --git a/public/app/plugins/datasource/influxdb/datasource.ts b/public/app/plugins/datasource/influxdb/datasource.ts index 4c2b1b034db..e63e002e7b3 100644 --- a/public/app/plugins/datasource/influxdb/datasource.ts +++ b/public/app/plugins/datasource/influxdb/datasource.ts @@ -17,6 +17,7 @@ import { FieldType, MetricFindValue, QueryResultMeta, + QueryVariableModel, RawTimeRange, ScopedVars, TIME_SERIES_TIME_FIELD_NAME, @@ -31,7 +32,6 @@ import { frameToMetricFindValue, getBackendSrv, } from '@grafana/runtime'; -import { CustomFormatterVariable } from '@grafana/scenes'; import { QueryFormat, SQLQuery } from '@grafana/sql'; import config from 'app/core/config'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; @@ -213,7 +213,12 @@ export default class InfluxDatasource extends DataSourceWithBackend + this.interpolateQueryExpr(value, variable, query.query) + ), // The raw query text }; } @@ -231,9 +236,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { return { ...groupBy, - params: groupBy.params?.map((param) => { - return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr); - }), + params: groupBy.params?.map((param) => this.templateSrv.replace(param.toString(), undefined)), }; }); } @@ -243,9 +246,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { return { ...select, - params: select.params?.map((param) => { - return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr); - }), + params: select.params?.map((param) => this.templateSrv.replace(param.toString(), undefined)), }; }); }); @@ -255,8 +256,8 @@ export default class InfluxDatasource extends DataSourceWithBackend { return { ...tag, - key: this.templateSrv.replace(tag.key, scopedVars, this.interpolateQueryExpr), - value: this.templateSrv.replace(tag.value, scopedVars, this.interpolateQueryExpr), + key: this.templateSrv.replace(tag.key, scopedVars), + value: this.templateSrv.replace(tag.value, scopedVars, 'pipe'), }; }); } @@ -264,34 +265,61 @@ export default class InfluxDatasource extends DataSourceWithBackend + this.interpolateQueryExpr(value, variable, query.query) + ), // The raw sql query text + rawSql: this.templateSrv.replace( + query.rawSql ?? '', + scopedVars, + (value: string | string[] = [], variable: QueryVariableModel) => + this.interpolateQueryExpr(value, variable, query.rawSql) + ), // The raw sql query text alias: this.templateSrv.replace(query.alias ?? '', scopedVars), - limit: this.templateSrv.replace(query.limit?.toString() ?? '', scopedVars, this.interpolateQueryExpr), - measurement: this.templateSrv.replace(query.measurement ?? '', scopedVars, this.interpolateQueryExpr), - policy: this.templateSrv.replace(query.policy ?? '', scopedVars, this.interpolateQueryExpr), - slimit: this.templateSrv.replace(query.slimit?.toString() ?? '', scopedVars, this.interpolateQueryExpr), + limit: this.templateSrv.replace(query.limit?.toString() ?? '', scopedVars), + measurement: this.templateSrv.replace(query.measurement ?? '', scopedVars), + policy: this.templateSrv.replace(query.policy ?? '', scopedVars), + slimit: this.templateSrv.replace(query.slimit?.toString() ?? '', scopedVars), tz: this.templateSrv.replace(query.tz ?? '', scopedVars), }; } - interpolateQueryExpr(value: string | string[] = [], variable: Partial) { - // if no multi or include all do not regexEscape - if (!variable.multi && !variable.includeAll) { - return influxRegularEscape(value); + // this should only be used for rawQueries. + interpolateQueryExpr(value: string | string[] = [], variable: QueryVariableModel, query?: string) { + // If there is no query just return the value directly + if (!query) { + return value; } - if (typeof value === 'string') { - return influxSpecialRegexEscape(value); + // If template variable is a multi-value variable + // we always want to deal with special chars. + if (variable.multi) { + if (typeof value === 'string') { + return escapeRegex(value); + } + + // If the value is a string array first escape them then join them with pipe + return value.map((v) => escapeRegex(v)).join('|'); } - const escapedValues = value.map((val) => influxSpecialRegexEscape(val)); + // If the variable is not a multi-value variable + // we want to see how it's been used. If it is used in a regex expression + // we escape it. Otherwise, we return it directly. + // regex below checks if the variable inside /^...$/ (^ and $ is optional) + // i.e. /^$myVar$/ or /$myVar/ + const regex = new RegExp(`\\/(?:\\^)?\\$${variable.name}(?:\\$)?\\/`, 'gm'); + if (regex.test(query)) { + if (typeof value === 'string') { + return escapeRegex(value); + } - if (escapedValues.length === 1) { - return escapedValues[0]; + // If the value is a string array first escape them then join them with pipe + return value.map((v) => escapeRegex(v)).join('|'); } - return escapedValues.join('|'); + return value; } async runMetadataQuery(target: InfluxQuery): Promise { @@ -322,7 +350,11 @@ export default class InfluxDatasource extends DataSourceWithBackend this.interpolateQueryExpr(value, variable, query) + ); return lastValueFrom(this._seriesQuery(interpolated, options)).then((resp) => { return this.responseParser.parse(query, resp); @@ -668,7 +700,12 @@ export default class InfluxDatasource extends DataSourceWithBackend + this.interpolateQueryExpr(value, variable, annotation.query) + ), rawQuery: true, }; @@ -696,7 +733,9 @@ export default class InfluxDatasource extends DataSourceWithBackend + this.interpolateQueryExpr(value, variable, query) + ); return lastValueFrom(this._seriesQuery(query, options)).then((data) => { if (!data || !data.results || !data.results[0]) { @@ -780,23 +819,3 @@ function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame { length: values.length, }; } - -export function influxRegularEscape(value: string | string[]) { - if (typeof value === 'string') { - // Check the value is a number. If not run to escape special characters - if (isNaN(parseFloat(value))) { - return escapeRegex(value); - } - } - - return value; -} - -export function influxSpecialRegexEscape(value: string | string[]) { - if (typeof value !== 'string') { - return value; - } - value = value.replace(/\\/g, '\\\\\\\\'); - value = value.replace(/[$^*{}\[\]\'+?.()|]/g, '$&'); - return value; -} diff --git a/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts b/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts index 5cad53c98ae..d48b4faf382 100644 --- a/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts +++ b/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts @@ -5,6 +5,7 @@ import { FetchResponse } from '@grafana/runtime/src'; import config from 'app/core/config'; import { TemplateSrv } from '../../../features/templating/template_srv'; +import { queryBuilder } from '../../../features/variables/shared/testing/builders'; import InfluxDatasource from './datasource'; import { @@ -149,6 +150,7 @@ describe('InfluxDataSource Backend Mode', () => { const text2 = 'interpolationText2'; const textWithoutFormatRegex = 'interpolationText,interpolationText2'; const textWithFormatRegex = 'interpolationText,interpolationText2'; + const justText = 'interpolationText'; const variableMap: Record = { $interpolationVar: text, $interpolationVar2: text2, @@ -178,14 +180,14 @@ describe('InfluxDataSource Backend Mode', () => { function influxChecks(query: InfluxQuery) { expect(templateSrv.replace).toBeCalledTimes(12); expect(query.alias).toBe(text); - expect(query.measurement).toBe(textWithFormatRegex); - expect(query.policy).toBe(textWithFormatRegex); - expect(query.limit).toBe(textWithFormatRegex); - expect(query.slimit).toBe(textWithFormatRegex); + expect(query.measurement).toBe(justText); + expect(query.policy).toBe(justText); + expect(query.limit).toBe(justText); + expect(query.slimit).toBe(justText); expect(query.tz).toBe(text); expect(query.tags![0].value).toBe(textWithFormatRegex); - expect(query.groupBy![0].params![0]).toBe(textWithFormatRegex); - expect(query.select![0][0].params![0]).toBe(textWithFormatRegex); + expect(query.groupBy![0].params![0]).toBe(justText); + expect(query.select![0][0].params![0]).toBe(justText); expect(query.adhocFilters?.[0].key).toBe(adhocFilters[0].key); } @@ -216,7 +218,12 @@ describe('InfluxDataSource Backend Mode', () => { }); describe('variable interpolation with chained variables with backend mode', () => { - const mockTemplateService = new TemplateSrv(); + const variablesMock = [queryBuilder().withId('var1').withName('var1').withCurrent('var1').build()]; + const mockTemplateService = new TemplateSrv({ + getVariables: () => variablesMock, + getVariableWithName: (name: string) => variablesMock.filter((v) => v.name === name)[0], + getFilteredVariables: jest.fn(), + }); mockTemplateService.getAdhocFilters = jest.fn((_: string) => []); let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService); const fetchMockImpl = () => @@ -263,6 +270,7 @@ describe('InfluxDataSource Backend Mode', () => { }, }); const qe = `SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`; + expect(fetchMock).toHaveBeenCalled(); const qData = fetchMock.mock.calls[0][0].data.queries[0].query; expect(qData).toBe(qe); }); diff --git a/public/app/plugins/datasource/influxdb/mocks.ts b/public/app/plugins/datasource/influxdb/mocks.ts index 678c4ea283d..8aee8e79b5c 100644 --- a/public/app/plugins/datasource/influxdb/mocks.ts +++ b/public/app/plugins/datasource/influxdb/mocks.ts @@ -8,14 +8,14 @@ import { FieldType, PluginType, ScopedVars, -} from '@grafana/data/src'; +} from '@grafana/data'; import { BackendDataSourceResponse, FetchResponse, getBackendSrv, setBackendSrv, VariableInterpolation, -} from '@grafana/runtime/src'; +} from '@grafana/runtime'; import { SQLQuery } from '@grafana/sql'; import { TemplateSrv } from '../../../features/templating/template_srv'; From a416bd761dbd76b876ebbdca13894e7ef89d0692 Mon Sep 17 00:00:00 2001 From: Alex Khomenko Date: Thu, 1 Feb 2024 06:58:50 +0100 Subject: [PATCH 28/84] Chore: Remove Form usage from admin components (#81618) * Chore: Remove Form usage from AdminEditOrgPage * Chore: Remove Form usage from UserCreatePage.tsx * Chore: Transform LdapPage to FC * Chore: Remove Form usage from LdapPage.tsx --- .../app/features/admin/AdminEditOrgPage.tsx | 39 ++-- public/app/features/admin/UserCreatePage.tsx | 71 ++++--- public/app/features/admin/ldap/LdapPage.tsx | 175 +++++++++--------- 3 files changed, 142 insertions(+), 143 deletions(-) diff --git a/public/app/features/admin/AdminEditOrgPage.tsx b/public/app/features/admin/AdminEditOrgPage.tsx index e1cab49a294..910bc855251 100644 --- a/public/app/features/admin/AdminEditOrgPage.tsx +++ b/public/app/features/admin/AdminEditOrgPage.tsx @@ -1,8 +1,9 @@ import React, { useState, useEffect } from 'react'; +import { useForm } from 'react-hook-form'; import { useAsyncFn } from 'react-use'; import { NavModelItem } from '@grafana/data'; -import { Form, Field, Input, Button, Legend, Alert } from '@grafana/ui'; +import { Field, Input, Button, Legend, Alert } from '@grafana/ui'; import { Page } from 'app/core/components/Page/Page'; import { contextSrv } from 'app/core/core'; import { GrafanaRouteComponentProps } from 'app/core/navigation/types'; @@ -27,6 +28,11 @@ const AdminEditOrgPage = ({ match }: Props) => { const [totalPages, setTotalPages] = useState(1); const [orgState, fetchOrg] = useAsyncFn(() => getOrg(orgId), []); + const { + handleSubmit, + register, + formState: { errors }, + } = useForm(); const [, fetchOrgUsers] = useAsyncFn(async (page) => { const result = await getOrgUsers(orgId, page); @@ -45,8 +51,8 @@ const AdminEditOrgPage = ({ match }: Props) => { fetchOrgUsers(page); }, [fetchOrg, fetchOrgUsers, page]); - const onUpdateOrgName = async (name: string) => { - await updateOrgName(name, orgId); + const onUpdateOrgName = async ({ orgName }: OrgNameDTO) => { + await updateOrgName(orgName, orgId); }; const renderMissingPermissionMessage = () => ( @@ -82,21 +88,18 @@ const AdminEditOrgPage = ({ match }: Props) => { <> Edit organization {orgState.value && ( -
onUpdateOrgName(values.orgName)} - > - {({ register, errors }) => ( - <> - - - - - - )} -
+
+ + + + +
)}
diff --git a/public/app/features/admin/UserCreatePage.tsx b/public/app/features/admin/UserCreatePage.tsx index da72bd30d90..34bdee39089 100644 --- a/public/app/features/admin/UserCreatePage.tsx +++ b/public/app/features/admin/UserCreatePage.tsx @@ -1,9 +1,10 @@ import React, { useCallback } from 'react'; +import { useForm } from 'react-hook-form'; import { useHistory } from 'react-router-dom'; import { NavModelItem } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; -import { Form, Button, Input, Field } from '@grafana/ui'; +import { Button, Input, Field } from '@grafana/ui'; import { Page } from 'app/core/components/Page/Page'; interface UserDTO { @@ -24,6 +25,11 @@ const pageNav: NavModelItem = { const UserCreatePage = () => { const history = useHistory(); + const { + handleSubmit, + register, + formState: { errors }, + } = useForm({ mode: 'onBlur' }); const onSubmit = useCallback( async (data: UserDTO) => { @@ -37,45 +43,34 @@ const UserCreatePage = () => { return ( -
- {({ register, errors }) => { - return ( - <> - - - + + + + - - - + + + - - - - - value.trim() !== '' && value.length >= 4, - })} - type="password" - /> - - - - ); - }} -
+ + + + + value.trim() !== '' && value.length >= 4, + })} + type="password" + /> + + + ); diff --git a/public/app/features/admin/ldap/LdapPage.tsx b/public/app/features/admin/ldap/LdapPage.tsx index d61c7711ca1..d7bf34f6b87 100644 --- a/public/app/features/admin/ldap/LdapPage.tsx +++ b/public/app/features/admin/ldap/LdapPage.tsx @@ -1,9 +1,10 @@ -import React, { PureComponent } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; +import { useForm } from 'react-hook-form'; import { connect, ConnectedProps } from 'react-redux'; import { NavModelItem } from '@grafana/data'; import { featureEnabled } from '@grafana/runtime'; -import { Alert, Button, Field, Form, Input, Stack } from '@grafana/ui'; +import { Alert, Button, Field, Input, Stack } from '@grafana/ui'; import { Page } from 'app/core/components/Page/Page'; import { contextSrv } from 'app/core/core'; import { GrafanaRouteComponentProps } from 'app/core/navigation/types'; @@ -37,10 +38,6 @@ interface OwnProps extends GrafanaRouteComponentProps<{}, { username?: string }> userError?: LdapError; } -interface State { - isLoading: boolean; -} - interface FormModel { username: string; } @@ -52,101 +49,105 @@ const pageNav: NavModelItem = { id: 'LDAP', }; -export class LdapPage extends PureComponent { - state = { - isLoading: true, - }; +export const LdapPage = ({ + clearUserMappingInfo, + queryParams, + loadLdapState, + loadLdapSyncStatus, + loadUserMapping, + clearUserError, + ldapUser, + userError, + ldapError, + ldapSyncInfo, + ldapConnectionInfo, +}: Props) => { + const [isLoading, setIsLoading] = useState(true); + const { register, handleSubmit } = useForm(); - async componentDidMount() { - const { clearUserMappingInfo, queryParams } = this.props; - await clearUserMappingInfo(); - await this.fetchLDAPStatus(); + const fetchUserMapping = useCallback( + async (username: string) => { + return loadUserMapping(username); + }, + [loadUserMapping] + ); - if (queryParams.username) { - await this.fetchUserMapping(queryParams.username); + useEffect(() => { + const fetchLDAPStatus = async () => { + return Promise.all([loadLdapState(), loadLdapSyncStatus()]); + }; + + async function init() { + await clearUserMappingInfo(); + await fetchLDAPStatus(); + + if (queryParams.username) { + await fetchUserMapping(queryParams.username); + } + + setIsLoading(false); } - this.setState({ isLoading: false }); - } + init(); + }, [clearUserMappingInfo, fetchUserMapping, loadLdapState, loadLdapSyncStatus, queryParams]); - async fetchLDAPStatus() { - const { loadLdapState, loadLdapSyncStatus } = this.props; - return Promise.all([loadLdapState(), loadLdapSyncStatus()]); - } - - async fetchUserMapping(username: string) { - const { loadUserMapping } = this.props; - return await loadUserMapping(username); - } - - search = (username: string) => { + const search = ({ username }: FormModel) => { if (username) { - this.fetchUserMapping(username); + fetchUserMapping(username); } }; - onClearUserError = () => { - this.props.clearUserError(); + const onClearUserError = () => { + clearUserError(); }; - render() { - const { ldapUser, userError, ldapError, ldapSyncInfo, ldapConnectionInfo, queryParams } = this.props; - const { isLoading } = this.state; - const canReadLDAPUser = contextSrv.hasPermission(AccessControlAction.LDAPUsersRead); + const canReadLDAPUser = contextSrv.hasPermission(AccessControlAction.LDAPUsersRead); + return ( + + + + {ldapError && ldapError.title && ( + + {ldapError.body} + + )} - return ( - - - - {ldapError && ldapError.title && ( - - {ldapError.body} - - )} + - + {featureEnabled('ldapsync') && ldapSyncInfo && } - {featureEnabled('ldapsync') && ldapSyncInfo && } - - {canReadLDAPUser && ( -
-

Test user mapping

-
this.search(data.username)}> - {({ register }) => ( - - - Run - - } - /> - - )} -
- {userError && userError.title && ( - - {userError.body} - - )} - {ldapUser && } -
- )} - - - - ); - } -} + {canReadLDAPUser && ( +
+

Test user mapping

+
+ + + Run + + } + /> + +
+ {userError && userError.title && ( + + {userError.body} + + )} + {ldapUser && } +
+ )} + + + + ); +}; const mapStateToProps = (state: StoreState) => ({ ldapConnectionInfo: state.ldap.connectionInfo, From 1a51240dc7435910f79b29b025a79d81b6f99bf8 Mon Sep 17 00:00:00 2001 From: idafurjes <36131195+idafurjes@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:13:15 +0100 Subject: [PATCH 29/84] Remove some folderIDs from database test (#81643) * Remove some folderIDs from database test * Add folderUID to insertTestDashboardForPlugin --- .../dashboards/database/database_test.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pkg/services/dashboards/database/database_test.go b/pkg/services/dashboards/database/database_test.go index 487bf6a97e7..906c9776f5f 100644 --- a/pkg/services/dashboards/database/database_test.go +++ b/pkg/services/dashboards/database/database_test.go @@ -502,7 +502,6 @@ func TestIntegrationDashboardDataAccess(t *testing.T) { }) require.NoError(t, err) - // nolint:staticcheck count, err := dashboardStore.CountDashboardsInFolders(context.Background(), &dashboards.CountDashboardsInFolderRequest{FolderUIDs: []string{folder.UID}, OrgID: 1}) require.NoError(t, err) require.Equal(t, count, int64(0)) @@ -519,9 +518,10 @@ func TestIntegrationDashboardDataAccessGivenPluginWithImportedDashboards(t *test require.NoError(t, err) pluginId := "test-app" - appFolder := insertTestDashboardForPlugin(t, dashboardStore, "app-test", 1, 0, true, pluginId) - insertTestDashboardForPlugin(t, dashboardStore, "app-dash1", 1, appFolder.ID, false, pluginId) - insertTestDashboardForPlugin(t, dashboardStore, "app-dash2", 1, appFolder.ID, false, pluginId) + insertTestDashboardForPlugin(t, dashboardStore, "app-test", 1, "", true, pluginId) + insertTestDashboardForPlugin(t, dashboardStore, "app-test", 1, "", true, pluginId) + insertTestDashboardForPlugin(t, dashboardStore, "app-dash1", 1, "", false, pluginId) + insertTestDashboardForPlugin(t, dashboardStore, "app-dash2", 1, "", false, pluginId) query := dashboards.GetDashboardsByPluginIDQuery{ PluginID: pluginId, @@ -658,8 +658,7 @@ func TestGetExistingDashboardByTitleAndFolder(t *testing.T) { savedFolder := insertTestDashboard(t, dashboardStore, "test dash folder", 1, 0, "", true, "prod", "webapp") savedDash := insertTestDashboard(t, dashboardStore, "test dash", 1, savedFolder.ID, savedFolder.UID, false, "prod", "webapp") err = sqlStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { - // nolint:staticcheck - _, err = getExistingDashboardByTitleAndFolder(sess, &dashboards.Dashboard{Title: savedDash.Title, FolderID: savedFolder.ID, FolderUID: savedFolder.UID, OrgID: 1}, sqlStore.GetDialect(), false, false) + _, err = getExistingDashboardByTitleAndFolder(sess, &dashboards.Dashboard{Title: savedDash.Title, FolderUID: savedFolder.UID, OrgID: 1}, sqlStore.GetDialect(), false, false) return err }) require.ErrorIs(t, err, dashboards.ErrDashboardWithSameNameInFolderExists) @@ -713,8 +712,8 @@ func TestIntegrationFindDashboardsByTitle(t *testing.T) { SignedInUser: user, }) require.NoError(t, err) - // nolint:staticcheck - insertTestDashboard(t, dashboardStore, "dashboard under f0", orgID, f0.ID, f0.UID, false) + + insertTestDashboard(t, dashboardStore, "dashboard under f0", orgID, 0, f0.UID, false) subfolder, err := folderServiceWithFlagOn.Create(context.Background(), &folder.CreateFolderCommand{ OrgID: orgID, @@ -723,8 +722,8 @@ func TestIntegrationFindDashboardsByTitle(t *testing.T) { SignedInUser: user, }) require.NoError(t, err) - // nolint:staticcheck - insertTestDashboard(t, dashboardStore, "dashboard under subfolder", orgID, subfolder.ID, subfolder.UID, false) + + insertTestDashboard(t, dashboardStore, "dashboard under subfolder", orgID, 0, subfolder.UID, false) type res struct { title string @@ -1065,12 +1064,12 @@ func insertTestDashboard(t *testing.T, dashboardStore dashboards.Store, title st } func insertTestDashboardForPlugin(t *testing.T, dashboardStore dashboards.Store, title string, orgId int64, - folderId int64, isFolder bool, pluginId string) *dashboards.Dashboard { + folderUID string, isFolder bool, pluginId string) *dashboards.Dashboard { t.Helper() cmd := dashboards.SaveDashboardCommand{ - OrgID: orgId, - FolderID: folderId, // nolint:staticcheck - IsFolder: isFolder, + OrgID: orgId, + IsFolder: isFolder, + FolderUID: folderUID, Dashboard: simplejson.NewFromAny(map[string]interface{}{ "id": nil, "title": title, From 1606095a5cf61f189c6ef5b0472982616f082edc Mon Sep 17 00:00:00 2001 From: Joey <90795735+joey-grafana@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:48:04 +0000 Subject: [PATCH 30/84] Tempo: Remove unused code (#81568) * Remove unused code * Remove unused code from tests --- .betterer.results | 6 +- .../components/model/link-patterns.test.ts | 271 +----------------- .../components/model/link-patterns.tsx | 89 +----- .../TraceView/components/model/span.tsx | 25 -- 4 files changed, 4 insertions(+), 387 deletions(-) delete mode 100644 public/app/features/explore/TraceView/components/model/span.tsx diff --git a/.betterer.results b/.betterer.results index 9c7d33abd00..eb523e906fe 100644 --- a/.betterer.results +++ b/.betterer.results @@ -3475,8 +3475,7 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "0"] ], "public/app/features/explore/TraceView/components/model/link-patterns.test.ts:5381": [ - [0, 0, 0, "Unexpected any. Specify a different type.", "0"], - [0, 0, 0, "Unexpected any. Specify a different type.", "1"] + [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], "public/app/features/explore/TraceView/components/model/link-patterns.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], @@ -3490,8 +3489,7 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "8"], [0, 0, 0, "Do not use any type assertions.", "9"], [0, 0, 0, "Unexpected any. Specify a different type.", "10"], - [0, 0, 0, "Do not use any type assertions.", "11"], - [0, 0, 0, "Unexpected any. Specify a different type.", "12"] + [0, 0, 0, "Do not use any type assertions.", "11"] ], "public/app/features/explore/TraceView/components/model/transform-trace-data.tsx:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"] diff --git a/public/app/features/explore/TraceView/components/model/link-patterns.test.ts b/public/app/features/explore/TraceView/components/model/link-patterns.test.ts index d31063ffc7c..8ddcd4516f0 100644 --- a/public/app/features/explore/TraceView/components/model/link-patterns.test.ts +++ b/public/app/features/explore/TraceView/components/model/link-patterns.test.ts @@ -12,17 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { Trace, TraceLink, TraceSpan } from '../types'; +import { Trace } from '../types'; import { processTemplate, createTestFunction, - getParameterInArray, - getParameterInAncestor, processLinkPattern, ProcessedLinkPattern, - computeLinks, - createGetLinks, computeTraceLink, } from './link-patterns'; @@ -153,179 +149,6 @@ describe('createTestFunction()', () => { }); }); -describe('getParameterInArray()', () => { - const data = [ - { key: 'mykey', value: 'ok' }, - { key: 'otherkey', value: 'v' }, - ]; - - it('returns an entry that is present', () => { - expect(getParameterInArray('mykey', data)).toBe(data[0]); - expect(getParameterInArray('otherkey', data)).toBe(data[1]); - }); - - it('returns undefined when the entry cannot be found', () => { - expect(getParameterInArray('myotherkey', data)).toBeUndefined(); - }); - - it('returns undefined when there is no array', () => { - expect(getParameterInArray('otherkey')).toBeUndefined(); - expect(getParameterInArray('otherkey', null)).toBeUndefined(); - }); -}); - -describe('getParameterInAncestor()', () => { - const spans = [ - { - depth: 0, - process: { - tags: [ - { key: 'a', value: 'a7' }, - { key: 'b', value: 'b7' }, - { key: 'c', value: 'c7' }, - { key: 'd', value: 'd7' }, - { key: 'e', value: 'e7' }, - { key: 'f', value: 'f7' }, - { key: 'g', value: 'g7' }, - { key: 'h', value: 'h7' }, - ], - }, - tags: [ - { key: 'a', value: 'a6' }, - { key: 'b', value: 'b6' }, - { key: 'c', value: 'c6' }, - { key: 'd', value: 'd6' }, - { key: 'e', value: 'e6' }, - { key: 'f', value: 'f6' }, - { key: 'g', value: 'g6' }, - ], - }, - { - depth: 1, - process: { - tags: [ - { key: 'a', value: 'a5' }, - { key: 'b', value: 'b5' }, - { key: 'c', value: 'c5' }, - { key: 'd', value: 'd5' }, - { key: 'e', value: 'e5' }, - { key: 'f', value: 'f5' }, - ], - }, - tags: [ - { key: 'a', value: 'a4' }, - { key: 'b', value: 'b4' }, - { key: 'c', value: 'c4' }, - { key: 'd', value: 'd4' }, - { key: 'e', value: 'e4' }, - ], - }, - { - depth: 1, - process: { - tags: [ - { key: 'a', value: 'a3' }, - { key: 'b', value: 'b3' }, - { key: 'c', value: 'c3' }, - { key: 'd', value: 'd3' }, - ], - }, - tags: [ - { key: 'a', value: 'a2' }, - { key: 'b', value: 'b2' }, - { key: 'c', value: 'c2' }, - ], - }, - { - depth: 2, - process: { - tags: [ - { key: 'a', value: 'a1' }, - { key: 'b', value: 'b1' }, - ], - }, - tags: [{ key: 'a', value: 'a0' }], - }, - ] as TraceSpan[]; - - spans[1].references = [ - { - spanID: 's1', - traceID: 't2', - refType: 'CHILD_OF', - span: spans[0], - }, - ]; - spans[2].references = [ - { - spanID: 's1', - traceID: 't2', - refType: 'CHILD_OF', - span: spans[0], - }, - ]; - spans[3].references = [ - { - spanID: 's1', - traceID: 't2', - refType: 'CHILD_OF', - span: spans[2], - }, - ]; - - it('uses current span tags', () => { - expect(getParameterInAncestor('a', spans[3])).toEqual({ key: 'a', value: 'a0' }); - expect(getParameterInAncestor('a', spans[2])).toEqual({ key: 'a', value: 'a2' }); - expect(getParameterInAncestor('a', spans[1])).toEqual({ key: 'a', value: 'a4' }); - expect(getParameterInAncestor('a', spans[0])).toEqual({ key: 'a', value: 'a6' }); - }); - - it('uses current span process tags', () => { - expect(getParameterInAncestor('b', spans[3])).toEqual({ key: 'b', value: 'b1' }); - expect(getParameterInAncestor('d', spans[2])).toEqual({ key: 'd', value: 'd3' }); - expect(getParameterInAncestor('f', spans[1])).toEqual({ key: 'f', value: 'f5' }); - expect(getParameterInAncestor('h', spans[0])).toEqual({ key: 'h', value: 'h7' }); - }); - - it('uses parent span tags', () => { - expect(getParameterInAncestor('c', spans[3])).toEqual({ key: 'c', value: 'c2' }); - expect(getParameterInAncestor('e', spans[2])).toEqual({ key: 'e', value: 'e6' }); - expect(getParameterInAncestor('f', spans[2])).toEqual({ key: 'f', value: 'f6' }); - expect(getParameterInAncestor('g', spans[2])).toEqual({ key: 'g', value: 'g6' }); - expect(getParameterInAncestor('g', spans[1])).toEqual({ key: 'g', value: 'g6' }); - }); - - it('uses parent span process tags', () => { - expect(getParameterInAncestor('d', spans[3])).toEqual({ key: 'd', value: 'd3' }); - expect(getParameterInAncestor('h', spans[2])).toEqual({ key: 'h', value: 'h7' }); - expect(getParameterInAncestor('h', spans[1])).toEqual({ key: 'h', value: 'h7' }); - }); - - it('uses grand-parent span tags', () => { - expect(getParameterInAncestor('e', spans[3])).toEqual({ key: 'e', value: 'e6' }); - expect(getParameterInAncestor('f', spans[3])).toEqual({ key: 'f', value: 'f6' }); - expect(getParameterInAncestor('g', spans[3])).toEqual({ key: 'g', value: 'g6' }); - }); - - it('uses grand-parent process tags', () => { - expect(getParameterInAncestor('h', spans[3])).toEqual({ key: 'h', value: 'h7' }); - }); - - it('returns undefined when the entry cannot be found', () => { - expect(getParameterInAncestor('i', spans[3])).toBeUndefined(); - }); - - it('does not break if some tags are not defined', () => { - const spansWithUndefinedTags = [ - { - depth: 0, - process: {}, - }, - ] as TraceSpan[]; - expect(getParameterInAncestor('a', spansWithUndefinedTags[0])).toBeUndefined(); - }); -}); - describe('computeTraceLink()', () => { const linkPatterns = [ { @@ -359,95 +182,3 @@ describe('computeTraceLink()', () => { ]); }); }); - -describe('computeLinks()', () => { - const linkPatterns = [ - { - type: 'tags', - key: 'myKey', - url: 'http://example.com/?myKey=#{myKey}', - text: 'first link (#{myKey})', - }, - { - key: 'myOtherKey', - url: 'http://example.com/?myKey=#{myOtherKey}&myKey=#{myKey}', - text: 'second link (#{myOtherKey})', - }, - ].map(processLinkPattern) as ProcessedLinkPattern[]; - - const spans = [ - { depth: 0, process: {}, tags: [{ key: 'myKey', value: 'valueOfMyKey' }] }, - { depth: 1, process: {}, logs: [{ fields: [{ key: 'myOtherKey', value: 'valueOfMy+Other+Key' }] }] }, - ] as unknown as TraceSpan[]; - spans[1].references = [ - { - spanID: 's1', - traceID: 't2', - refType: 'CHILD_OF', - span: spans[0], - }, - ]; - - it('correctly computes links', () => { - expect(computeLinks(linkPatterns, spans[0], spans[0].tags, 0)).toEqual([ - { - url: 'http://example.com/?myKey=valueOfMyKey', - text: 'first link (valueOfMyKey)', - }, - ]); - expect(computeLinks(linkPatterns, spans[1], spans[1].logs[0].fields, 0)).toEqual([ - { - url: 'http://example.com/?myKey=valueOfMy%2BOther%2BKey&myKey=valueOfMyKey', - text: 'second link (valueOfMy+Other+Key)', - }, - ]); - }); -}); - -describe('getLinks()', () => { - const linkPatterns = [ - { - key: 'mySpecialKey', - url: 'http://example.com/?mySpecialKey=#{mySpecialKey}', - text: 'special key link (#{mySpecialKey})', - }, - ].map(processLinkPattern) as ProcessedLinkPattern[]; - const template = jest.spyOn(linkPatterns[0]!.url, 'template'); - - const span = { depth: 0, process: {}, tags: [{ key: 'mySpecialKey', value: 'valueOfMyKey' }] } as TraceSpan; - - let cache: WeakMap; - - beforeEach(() => { - cache = new WeakMap(); - template.mockClear(); - }); - - it('does not access the cache if there is no link pattern', () => { - cache.get = jest.fn(); - const getLinks = createGetLinks([], cache); - expect(getLinks(span, span.tags, 0)).toEqual([]); - expect(cache.get).not.toHaveBeenCalled(); - }); - - it('returns the result from the cache', () => { - const result: TraceLink[] = []; - cache.set(span.tags[0], result); - const getLinks = createGetLinks(linkPatterns, cache); - expect(getLinks(span, span.tags, 0)).toBe(result); - expect(template).not.toHaveBeenCalled(); - }); - - it('adds the result to the cache', () => { - const getLinks = createGetLinks(linkPatterns, cache); - const result = getLinks(span, span.tags, 0); - expect(template).toHaveBeenCalledTimes(1); - expect(result).toEqual([ - { - url: 'http://example.com/?mySpecialKey=valueOfMyKey', - text: 'special key link (valueOfMyKey)', - }, - ]); - expect(cache.get(span.tags[0])).toBe(result); - }); -}); diff --git a/public/app/features/explore/TraceView/components/model/link-patterns.tsx b/public/app/features/explore/TraceView/components/model/link-patterns.tsx index 930fc541056..cf52965244d 100644 --- a/public/app/features/explore/TraceView/components/model/link-patterns.tsx +++ b/public/app/features/explore/TraceView/components/model/link-patterns.tsx @@ -15,11 +15,9 @@ import { uniq as _uniq } from 'lodash'; import memoize from 'lru-memoize'; -import { TraceSpan, TraceLink, TraceKeyValuePair, Trace, TNil } from '../types'; +import { Trace } from '../types'; import { getConfigValue } from '../utils/config/get-config'; -import { getParent } from './span'; - const parameterRegExp = /#\{([^{}]*)\}/g; type ProcessedTemplate = { @@ -119,25 +117,6 @@ export function processLinkPattern(pattern: any): ProcessedLinkPattern | null { } } -export function getParameterInArray(name: string, array?: TraceKeyValuePair[] | TNil) { - if (array) { - return array.find((entry) => entry.key === name); - } - return undefined; -} - -export function getParameterInAncestor(name: string, span: TraceSpan) { - let currentSpan: TraceSpan | TNil = span; - while (currentSpan) { - const result = getParameterInArray(name, currentSpan.tags) || getParameterInArray(name, currentSpan.process.tags); - if (result) { - return result; - } - currentSpan = getParent(currentSpan); - } - return undefined; -} - function callTemplate(template: ProcessedTemplate, data: any) { return template.template(data); } @@ -173,70 +152,6 @@ export function computeTraceLink(linkPatterns: ProcessedLinkPattern[], trace: Tr return result; } -export function computeLinks( - linkPatterns: ProcessedLinkPattern[], - span: TraceSpan, - items: TraceKeyValuePair[], - itemIndex: number -) { - const item = items[itemIndex]; - let type = 'logs'; - const processTags = span.process.tags === items; - if (processTags) { - type = 'process'; - } - const spanTags = span.tags === items; - if (spanTags) { - type = 'tags'; - } - const result: Array<{ url: string; text: string }> = []; - linkPatterns.forEach((pattern) => { - if (pattern.type(type) && pattern.key(item.key) && pattern.value(item.value)) { - const parameterValues: Record = {}; - const allParameters = pattern.parameters.every((parameter) => { - let entry = getParameterInArray(parameter, items); - if (!entry && !processTags) { - // do not look in ancestors for process tags because the same object may appear in different places in the hierarchy - // and the cache in getLinks uses that object as a key - entry = getParameterInAncestor(parameter, span); - } - if (entry) { - parameterValues[parameter] = entry.value; - return true; - } - // eslint-disable-next-line no-console - console.warn( - `Skipping link pattern, missing parameter ${parameter} for key ${item.key} in ${type}.`, - pattern.object - ); - return false; - }); - if (allParameters) { - result.push({ - url: callTemplate(pattern.url, parameterValues), - text: callTemplate(pattern.text, parameterValues), - }); - } - } - }); - return result; -} - -export function createGetLinks(linkPatterns: ProcessedLinkPattern[], cache: WeakMap) { - return (span: TraceSpan, items: TraceKeyValuePair[], itemIndex: number) => { - if (linkPatterns.length === 0) { - return []; - } - const item = items[itemIndex]; - let result = cache.get(item); - if (!result) { - result = computeLinks(linkPatterns, span, items, itemIndex); - cache.set(item, result); - } - return result; - }; -} - const processedLinks = (getConfigValue('linkPatterns') || []) .map(processLinkPattern) .filter((link: ProcessedLinkPattern | null): link is ProcessedLinkPattern => Boolean(link)); @@ -248,5 +163,3 @@ export const getTraceLinks: (trace: Trace | undefined) => TLinksRV = memoize(10) } return computeTraceLink(processedLinks, trace); }); - -export default createGetLinks(processedLinks, new WeakMap()); diff --git a/public/app/features/explore/TraceView/components/model/span.tsx b/public/app/features/explore/TraceView/components/model/span.tsx deleted file mode 100644 index 07764898a24..00000000000 --- a/public/app/features/explore/TraceView/components/model/span.tsx +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2017 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { TraceSpan } from '../types'; - -/** - * Searches the span.references to find 'CHILD_OF' reference type or returns null. - * @param {TraceSpan} span The span whose parent is to be returned. - * @returns {TraceSpan|null} The parent span if there is one, null otherwise. - */ -export function getParent(span: TraceSpan) { - const parentRef = span.references ? span.references.find((ref) => ref.refType === 'CHILD_OF') : null; - return parentRef ? parentRef.span : null; -} From fd54d2549608b0747ae49706cbd7d5043f8083af Mon Sep 17 00:00:00 2001 From: Joey <90795735+joey-grafana@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:48:18 +0000 Subject: [PATCH 31/84] Tempo: Fix issue with button click area being too large (#81500) Fix issue with button click area being too large --- .../datasource/tempo/traceql/QueryEditor.tsx | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/public/app/plugins/datasource/tempo/traceql/QueryEditor.tsx b/public/app/plugins/datasource/tempo/traceql/QueryEditor.tsx index c4e9f977f22..21b4cfc6cd1 100644 --- a/public/app/plugins/datasource/tempo/traceql/QueryEditor.tsx +++ b/public/app/plugins/datasource/tempo/traceql/QueryEditor.tsx @@ -2,7 +2,7 @@ import { css } from '@emotion/css'; import { defaults } from 'lodash'; import React, { useRef, useState } from 'react'; -import { QueryEditorProps } from '@grafana/data'; +import { GrafanaTheme2, QueryEditorProps } from '@grafana/data'; import { config, reportInteraction } from '@grafana/runtime'; import { Button, InlineLabel, useStyles2 } from '@grafana/ui'; @@ -46,32 +46,30 @@ export function QueryEditor(props: Props) { {!showCopyFromSearchButton && ( - -
- Continue editing the query from the Search tab? - -
- + props.onClearResults(); + props.onChange({ + ...query, + query: generateQueryFromFilters(query.filters || []), + }); + setShowCopyFromSearchButton(true); + }} + style={{ marginLeft: '10px' }} + > + Copy query from Search + +
)} ({ +const getStyles = (theme: GrafanaTheme2) => ({ optionsContainer: css({ marginTop: '10px', }), + copyContainer: css({ + backgroundColor: theme.colors.background.secondary, + padding: theme.spacing(0.5, 1), + fontSize: theme.typography.body.fontSize, + }), }); From 4561863fca6e4ffe1567963b071e681f6276525d Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 1 Feb 2024 11:17:38 +0100 Subject: [PATCH 32/84] Chore: Remove unused header params in prometheus request (#81684) Remove unused header params --- pkg/tsdb/prometheus/querydata/request.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/tsdb/prometheus/querydata/request.go b/pkg/tsdb/prometheus/querydata/request.go index c35bede0815..ceb5e3877d9 100644 --- a/pkg/tsdb/prometheus/querydata/request.go +++ b/pkg/tsdb/prometheus/querydata/request.go @@ -101,7 +101,7 @@ func (s *QueryData) Execute(ctx context.Context, req *backend.QueryDataRequest) return &result, err } - r := s.fetch(ctx, s.client, query, req.Headers) + r := s.fetch(ctx, s.client, query) if r == nil { s.log.FromContext(ctx).Debug("Received nil response from runQuery", "query", query.Expr) continue @@ -112,7 +112,7 @@ func (s *QueryData) Execute(ctx context.Context, req *backend.QueryDataRequest) return &result, nil } -func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models.Query, headers map[string]string) *backend.DataResponse { +func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models.Query) *backend.DataResponse { traceCtx, end := s.trace(ctx, q) defer end() @@ -125,14 +125,14 @@ func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models. } if q.InstantQuery { - res := s.instantQuery(traceCtx, client, q, headers) + res := s.instantQuery(traceCtx, client, q) dr.Error = res.Error dr.Frames = res.Frames dr.Status = res.Status } if q.RangeQuery { - res := s.rangeQuery(traceCtx, client, q, headers) + res := s.rangeQuery(traceCtx, client, q) if res.Error != nil { if dr.Error == nil { dr.Error = res.Error @@ -147,7 +147,7 @@ func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models. } if q.ExemplarQuery { - res := s.exemplarQuery(traceCtx, client, q, headers) + res := s.exemplarQuery(traceCtx, client, q) if res.Error != nil { // If exemplar query returns error, we want to only log it and // continue with other results processing @@ -159,7 +159,7 @@ func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models. return dr } -func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query, headers map[string]string) backend.DataResponse { +func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query) backend.DataResponse { res, err := c.QueryRange(ctx, q) if err != nil { return backend.DataResponse{ @@ -178,7 +178,7 @@ func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models. return s.parseResponse(ctx, q, res) } -func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query, headers map[string]string) backend.DataResponse { +func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query) backend.DataResponse { res, err := c.QueryInstant(ctx, q) if err != nil { return backend.DataResponse{ @@ -204,7 +204,7 @@ func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *model return s.parseResponse(ctx, q, res) } -func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query, headers map[string]string) backend.DataResponse { +func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query) backend.DataResponse { res, err := c.QueryExemplars(ctx, q) if err != nil { return backend.DataResponse{ From c43a170009e1fd6b229f2eea15621b8d7b78d09e Mon Sep 17 00:00:00 2001 From: Mihai Doarna Date: Thu, 1 Feb 2024 12:27:39 +0200 Subject: [PATCH 33/84] SSO Auth: fix typos on field description (#81685) fix typos on field description --- public/app/features/auth-config/fields.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/app/features/auth-config/fields.tsx b/public/app/features/auth-config/fields.tsx index d5fddfc55fe..60431a4b8ea 100644 --- a/public/app/features/auth-config/fields.tsx +++ b/public/app/features/auth-config/fields.tsx @@ -313,17 +313,17 @@ export function fieldMap(provider: string): Record { }, tlsClientCa: { label: 'TLS client ca', - description: 'The path to the trusted certificate authority list. Does not applicable on Grafana Cloud.', + description: 'The path to the trusted certificate authority list. Is not applicable on Grafana Cloud.', type: 'text', }, tlsClientCert: { label: 'TLS client cert', - description: 'The path to the certificate. Does not applicable on Grafana Cloud.', + description: 'The path to the certificate. Is not applicable on Grafana Cloud.', type: 'text', }, tlsClientKey: { label: 'TLS client key', - description: 'The path to the key. Does not applicable on Grafana Cloud.', + description: 'The path to the key. Is not applicable on Grafana Cloud.', type: 'text', }, tlsSkipVerifyInsecure: { @@ -338,7 +338,7 @@ export function fieldMap(provider: string): Record { label: 'Groups attribute path', description: 'JMESPath expression to use for user group lookup. If you configure allowed_groups, \n' + - 'you must also configure groupsƒ_attribute_path.', + 'you must also configure groups_attribute_path.', type: 'text', }, teamsUrl: { From 536153c3362782ae7f79e06755e8c7e0cd3b3acd Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 1 Feb 2024 11:58:24 +0100 Subject: [PATCH 34/84] InfluxDB: Run queries in parallel behind influxdbRunQueriesInParallel feature toggle (#81209) * create the feature flag * bring the concurrency in to the play * Update feature flag * Use concurrency number from settings * update influxdb dependency * use ConcurrentQueryCount from plugin-sdk-go * use helper method for concurrent query count * log the error * add value guard * add unit tests * handle concurrency error --- .../src/types/featureToggles.gen.ts | 1 + pkg/plugins/config/config.go | 5 +- pkg/plugins/envvars/envvars.go | 26 ++--- pkg/plugins/envvars/envvars_test.go | 27 ++++++ pkg/services/featuremgmt/registry.go | 8 ++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + .../pluginsintegration/config/config.go | 1 + pkg/tsdb/influxdb/influxql/influxql.go | 96 +++++++++++++++---- 9 files changed, 136 insertions(+), 33 deletions(-) diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 243dc60d572..fab1a2a7a62 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -63,6 +63,7 @@ export interface FeatureToggles { prometheusMetricEncyclopedia?: boolean; influxdbBackendMigration?: boolean; influxqlStreamingParser?: boolean; + influxdbRunQueriesInParallel?: boolean; clientTokenRotation?: boolean; prometheusDataplane?: boolean; lokiMetricDataplane?: boolean; diff --git a/pkg/plugins/config/config.go b/pkg/plugins/config/config.go index bedd5cd45f4..e3611a86a13 100644 --- a/pkg/plugins/config/config.go +++ b/pkg/plugins/config/config.go @@ -48,12 +48,14 @@ type Cfg struct { AngularSupportEnabled bool HideAngularDeprecation []string + + ConcurrentQueryCount int } func NewCfg(devMode bool, pluginsPath string, pluginSettings setting.PluginSettings, pluginsAllowUnsigned []string, awsAllowedAuthProviders []string, awsAssumeRoleEnabled bool, awsExternalId string, azure *azsettings.AzureSettings, secureSocksDSProxy setting.SecureSocksDSProxySettings, grafanaVersion string, logDatasourceRequests bool, pluginsCDNURLTemplate string, appURL string, appSubURL string, tracing Tracing, features featuremgmt.FeatureToggles, angularSupportEnabled bool, - grafanaComURL string, disablePlugins []string, hideAngularDeprecation []string, forwardHostEnvVars []string) *Cfg { + grafanaComURL string, disablePlugins []string, hideAngularDeprecation []string, forwardHostEnvVars []string, concurrentQueryCount int) *Cfg { return &Cfg{ log: log.New("plugin.cfg"), PluginsPath: pluginsPath, @@ -77,5 +79,6 @@ func NewCfg(devMode bool, pluginsPath string, pluginSettings setting.PluginSetti AngularSupportEnabled: angularSupportEnabled, HideAngularDeprecation: hideAngularDeprecation, ForwardHostEnvVars: forwardHostEnvVars, + ConcurrentQueryCount: concurrentQueryCount, } } diff --git a/pkg/plugins/envvars/envvars.go b/pkg/plugins/envvars/envvars.go index 9fc106113eb..624ed773fc4 100644 --- a/pkg/plugins/envvars/envvars.go +++ b/pkg/plugins/envvars/envvars.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/proxy" "github.com/grafana/grafana-plugin-sdk-go/experimental/featuretoggles" + "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/auth" "github.com/grafana/grafana/pkg/plugins/config" @@ -105,14 +106,17 @@ func (s *Service) GetConfigMap(ctx context.Context, pluginID string, _ *auth.Ext if s.cfg.GrafanaAppURL != "" { m[backend.AppURL] = s.cfg.GrafanaAppURL } + if s.cfg.ConcurrentQueryCount != 0 { + m[backend.ConcurrentQueryCount] = strconv.Itoa(s.cfg.ConcurrentQueryCount) + } // TODO add support via plugin SDK - //if externalService != nil { + // if externalService != nil { // m[oauthtokenretriever.AppURL] = s.cfg.GrafanaAppURL // m[oauthtokenretriever.AppClientID] = externalService.ClientID // m[oauthtokenretriever.AppClientSecret] = externalService.ClientSecret // m[oauthtokenretriever.AppPrivateKey] = externalService.PrivateKey - //} + // } if s.cfg.Features != nil { enabledFeatures := s.cfg.Features.GetEnabled(ctx) @@ -126,15 +130,15 @@ func (s *Service) GetConfigMap(ctx context.Context, pluginID string, _ *auth.Ext } } // TODO add support via plugin SDK - //if s.cfg.AWSAssumeRoleEnabled { + // if s.cfg.AWSAssumeRoleEnabled { // m[awsds.AssumeRoleEnabledEnvVarKeyName] = "true" - //} - //if len(s.cfg.AWSAllowedAuthProviders) > 0 { + // } + // if len(s.cfg.AWSAllowedAuthProviders) > 0 { // m[awsds.AllowedAuthProvidersEnvVarKeyName] = strings.Join(s.cfg.AWSAllowedAuthProviders, ",") - //} - //if s.cfg.AWSExternalId != "" { + // } + // if s.cfg.AWSExternalId != "" { // m[awsds.GrafanaAssumeRoleExternalIdKeyName] = s.cfg.AWSExternalId - //} + // } if s.cfg.ProxySettings.Enabled { m[proxy.PluginSecureSocksProxyEnabled] = "true" @@ -198,10 +202,10 @@ func (s *Service) GetConfigMap(ctx context.Context, pluginID string, _ *auth.Ext } // TODO add support via plugin SDK - //ps := getPluginSettings(pluginID, s.cfg) - //for k, v := range ps { + // ps := getPluginSettings(pluginID, s.cfg) + // for k, v := range ps { // m[fmt.Sprintf("%s_%s", customConfigPrefix, strings.ToUpper(k))] = v - //} + // } return m } diff --git a/pkg/plugins/envvars/envvars_test.go b/pkg/plugins/envvars/envvars_test.go index 7ed2c968f33..2e93f4cba1a 100644 --- a/pkg/plugins/envvars/envvars_test.go +++ b/pkg/plugins/envvars/envvars_test.go @@ -798,6 +798,33 @@ func TestService_GetConfigMap_appURL(t *testing.T) { }) } +func TestService_GetConfigMap_concurrentQueryCount(t *testing.T) { + t.Run("Uses the configured concurrent query count", func(t *testing.T) { + s := &Service{ + cfg: &config.Cfg{ + ConcurrentQueryCount: 42, + }, + } + require.Equal(t, map[string]string{"GF_CONCURRENT_QUERY_COUNT": "42"}, s.GetConfigMap(context.Background(), "", nil)) + }) + + t.Run("Doesn't set the concurrent query count if it is not in the config", func(t *testing.T) { + s := &Service{ + cfg: &config.Cfg{}, + } + require.Equal(t, map[string]string{}, s.GetConfigMap(context.Background(), "", nil)) + }) + + t.Run("Doesn't set the concurrent query count if it is zero", func(t *testing.T) { + s := &Service{ + cfg: &config.Cfg{ + ConcurrentQueryCount: 0, + }, + } + require.Equal(t, map[string]string{}, s.GetConfigMap(context.Background(), "", nil)) + }) +} + func TestService_GetConfigMap_azure(t *testing.T) { azSettings := &azsettings.AzureSettings{ Cloud: azsettings.AzurePublic, diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index f790618239a..20559900542 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -389,6 +389,14 @@ var ( Owner: grafanaObservabilityMetricsSquad, Created: time.Date(2023, time.November, 29, 12, 0, 0, 0, time.UTC), }, + { + Name: "influxdbRunQueriesInParallel", + Description: "Enables running InfluxDB Influxql queries in parallel", + Stage: FeatureStagePrivatePreview, + FrontendOnly: false, + Owner: grafanaObservabilityMetricsSquad, + Created: time.Date(2024, time.January, 29, 12, 0, 0, 0, time.UTC), + }, { Name: "clientTokenRotation", Description: "Replaces the current in-request token rotation so that the client initiates the rotation", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index ac27e659c64..694c8a3e491 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -44,6 +44,7 @@ individualCookiePreferences,experimental,@grafana/backend-platform,2023-02-23,fa prometheusMetricEncyclopedia,GA,@grafana/observability-metrics,2023-03-07,false,false,true influxdbBackendMigration,GA,@grafana/observability-metrics,2023-03-15,false,false,true influxqlStreamingParser,experimental,@grafana/observability-metrics,2023-11-29,false,false,false +influxdbRunQueriesInParallel,privatePreview,@grafana/observability-metrics,2024-01-29,false,false,false clientTokenRotation,GA,@grafana/identity-access-team,2023-03-23,false,false,false prometheusDataplane,GA,@grafana/observability-metrics,2023-03-29,false,false,false lokiMetricDataplane,GA,@grafana/observability-logs,2023-04-13,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 42b161e6701..640d999ac1a 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -187,6 +187,10 @@ const ( // Enable streaming JSON parser for InfluxDB datasource InfluxQL query language FlagInfluxqlStreamingParser = "influxqlStreamingParser" + // FlagInfluxdbRunQueriesInParallel + // Enables running InfluxDB Influxql queries in parallel + FlagInfluxdbRunQueriesInParallel = "influxdbRunQueriesInParallel" + // FlagClientTokenRotation // Replaces the current in-request token rotation so that the client initiates the rotation FlagClientTokenRotation = "clientTokenRotation" diff --git a/pkg/services/pluginsintegration/config/config.go b/pkg/services/pluginsintegration/config/config.go index 50056ce54f3..9cd7a54b1aa 100644 --- a/pkg/services/pluginsintegration/config/config.go +++ b/pkg/services/pluginsintegration/config/config.go @@ -49,6 +49,7 @@ func ProvideConfig(settingProvider setting.Provider, grafanaCfg *setting.Cfg, fe grafanaCfg.DisablePlugins, grafanaCfg.HideAngularDeprecation, grafanaCfg.ForwardHostEnvVars, + grafanaCfg.ConcurrentQueryCount, ), nil } diff --git a/pkg/tsdb/influxdb/influxql/influxql.go b/pkg/tsdb/influxdb/influxql/influxql.go index f6cdf46aef5..88a7ac7b1bd 100644 --- a/pkg/tsdb/influxdb/influxql/influxql.go +++ b/pkg/tsdb/influxdb/influxql/influxql.go @@ -3,11 +3,14 @@ package influxql import ( "context" "errors" + "fmt" "net/http" "net/url" "path" "strings" + "sync" + "github.com/grafana/dskit/concurrency" "github.com/grafana/grafana-plugin-sdk-go/backend" "go.opentelemetry.io/otel/trace" @@ -30,40 +33,91 @@ var ( func Query(ctx context.Context, tracer trace.Tracer, dsInfo *models.DatasourceInfo, req *backend.QueryDataRequest, features featuremgmt.FeatureToggles) (*backend.QueryDataResponse, error) { logger := glog.FromContext(ctx) response := backend.NewQueryDataResponse() + var err error - for _, reqQuery := range req.Queries { - query, err := models.QueryParse(reqQuery) + // We are testing running of queries in parallel behind feature flag + if features.IsEnabled(ctx, featuremgmt.FlagInfluxdbRunQueriesInParallel) { + concurrentQueryCount, err := req.PluginContext.GrafanaConfig.ConcurrentQueryCount() if err != nil { - return &backend.QueryDataResponse{}, err + logger.Debug(fmt.Sprintf("Concurrent Query Count read/parse error: %v", err), featuremgmt.FlagInfluxdbRunQueriesInParallel) + concurrentQueryCount = 10 } - rawQuery, err := query.Build(req) - if err != nil { - return &backend.QueryDataResponse{}, err - } + responseLock := sync.Mutex{} + err = concurrency.ForEachJob(ctx, len(req.Queries), concurrentQueryCount, func(ctx context.Context, idx int) error { + reqQuery := req.Queries[idx] + query, err := models.QueryParse(reqQuery) + if err != nil { + return err + } - query.RefID = reqQuery.RefID - query.RawQuery = rawQuery + rawQuery, err := query.Build(req) + if err != nil { + return err + } - if setting.Env == setting.Dev { - logger.Debug("Influxdb query", "raw query", rawQuery) - } + query.RefID = reqQuery.RefID + query.RawQuery = rawQuery - request, err := createRequest(ctx, logger, dsInfo, rawQuery, query.Policy) - if err != nil { - return &backend.QueryDataResponse{}, err - } + if setting.Env == setting.Dev { + logger.Debug("Influxdb query", "raw query", rawQuery) + } - resp, err := execute(ctx, tracer, dsInfo, logger, query, request, features.IsEnabled(ctx, featuremgmt.FlagInfluxqlStreamingParser)) + request, err := createRequest(ctx, logger, dsInfo, rawQuery, query.Policy) + if err != nil { + return err + } + + resp, err := execute(ctx, tracer, dsInfo, logger, query, request, features.IsEnabled(ctx, featuremgmt.FlagInfluxqlStreamingParser)) + + responseLock.Lock() + defer responseLock.Unlock() + if err != nil { + response.Responses[query.RefID] = backend.DataResponse{Error: err} + } else { + response.Responses[query.RefID] = resp + } + return nil // errors are saved per-query,always return nil + }) if err != nil { - response.Responses[query.RefID] = backend.DataResponse{Error: err} - } else { - response.Responses[query.RefID] = resp + logger.Debug("Influxdb concurrent query error", "concurrent query", err) + } + } else { + for _, reqQuery := range req.Queries { + query, err := models.QueryParse(reqQuery) + if err != nil { + return &backend.QueryDataResponse{}, err + } + + rawQuery, err := query.Build(req) + if err != nil { + return &backend.QueryDataResponse{}, err + } + + query.RefID = reqQuery.RefID + query.RawQuery = rawQuery + + if setting.Env == setting.Dev { + logger.Debug("Influxdb query", "raw query", rawQuery) + } + + request, err := createRequest(ctx, logger, dsInfo, rawQuery, query.Policy) + if err != nil { + return &backend.QueryDataResponse{}, err + } + + resp, err := execute(ctx, tracer, dsInfo, logger, query, request, features.IsEnabled(ctx, featuremgmt.FlagInfluxqlStreamingParser)) + + if err != nil { + response.Responses[query.RefID] = backend.DataResponse{Error: err} + } else { + response.Responses[query.RefID] = resp + } } } - return response, nil + return response, err } func createRequest(ctx context.Context, logger log.Logger, dsInfo *models.DatasourceInfo, queryStr string, retentionPolicy string) (*http.Request, error) { From 0f1ba3a9feaffad633258d05806c51b28c34aa1a Mon Sep 17 00:00:00 2001 From: Joao Silva <100691367+JoaoSilvaGrafana@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:59:47 +0000 Subject: [PATCH 35/84] Chore: Change codeowners of annotations folder (#81551) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 43b600a7345..166112a5a7d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -382,7 +382,7 @@ /public/app/features/all.ts @grafana/grafana-frontend-platform /public/app/features/admin/ @grafana/identity-access-team /public/app/features/auth-config/ @grafana/identity-access-team -/public/app/features/annotations/ @grafana/grafana-frontend-platform +/public/app/features/annotations/ @grafana/dashboards-squad /public/app/features/api-keys/ @grafana/identity-access-team /public/app/features/canvas/ @grafana/dataviz-squad /public/app/features/geo/ @grafana/dataviz-squad From 3df0611f81114268c706d1ae2ade9c761f71e35f Mon Sep 17 00:00:00 2001 From: Gabriel MABILLE Date: Thu, 1 Feb 2024 12:37:01 +0100 Subject: [PATCH 36/84] RBAC: Fix authorize in org (#81552) * RBAC: Fix authorize in org * Implement option 2 * Fix typo * Fix alerting test * Add test to cover the not member case --- .../accesscontrol/acimpl/accesscontrol.go | 12 +++++++--- .../accesscontrol/authorize_in_org_test.go | 18 +++++++++++++++ pkg/services/accesscontrol/middleware.go | 23 +++++++++++++------ pkg/services/accesscontrol/models.go | 1 + pkg/services/auth/identity/requester.go | 2 ++ pkg/services/authn/identity.go | 14 +++++++++++ pkg/services/user/identity.go | 17 ++++++++++++++ .../api/alerting/api_backtesting_test.go | 4 +++- 8 files changed, 80 insertions(+), 11 deletions(-) diff --git a/pkg/services/accesscontrol/acimpl/accesscontrol.go b/pkg/services/accesscontrol/acimpl/accesscontrol.go index f99e4edbe82..f5d8774292d 100644 --- a/pkg/services/accesscontrol/acimpl/accesscontrol.go +++ b/pkg/services/accesscontrol/acimpl/accesscontrol.go @@ -39,13 +39,19 @@ func (a *AccessControl) Evaluate(ctx context.Context, user identity.Requester, e } namespace, identifier := user.GetNamespacedID() - if len(user.GetPermissions()) == 0 { + + // If the user is in no organization, then the evaluation must happen based on the user's global permissions + permissions := user.GetPermissions() + if user.GetOrgID() == accesscontrol.NoOrgID { + permissions = user.GetGlobalPermissions() + } + if len(permissions) == 0 { a.log.Debug("No permissions set for entity", "namespace", namespace, "id", identifier, "orgID", user.GetOrgID(), "login", user.GetLogin()) return false, nil } // Test evaluation without scope resolver first, this will prevent 403 for wildcard scopes when resource does not exist - if evaluator.Evaluate(user.GetPermissions()) { + if evaluator.Evaluate(permissions) { return true, nil } @@ -57,7 +63,7 @@ func (a *AccessControl) Evaluate(ctx context.Context, user identity.Requester, e return false, err } - return resolvedEvaluator.Evaluate(user.GetPermissions()), nil + return resolvedEvaluator.Evaluate(permissions), nil } func (a *AccessControl) RegisterScopeAttributeResolver(prefix string, resolver accesscontrol.ScopeAttributeResolver) { diff --git a/pkg/services/accesscontrol/authorize_in_org_test.go b/pkg/services/accesscontrol/authorize_in_org_test.go index 812f56634fa..8a9f6d2b814 100644 --- a/pkg/services/accesscontrol/authorize_in_org_test.go +++ b/pkg/services/accesscontrol/authorize_in_org_test.go @@ -202,6 +202,24 @@ func TestAuthorizeInOrgMiddleware(t *testing.T) { teamService: &teamtest.FakeService{}, expectedStatus: http.StatusForbidden, }, + { + name: "should fetch global user permissions when user is not a member of the target org", + orgIDGetter: func(c *contextmodel.ReqContext) (int64, error) { + return 2, nil + }, + evaluator: accesscontrol.EvalPermission("users:read", "users:*"), + accessControl: ac, + acService: &actest.FakeService{ + ExpectedPermissions: []accesscontrol.Permission{{Action: "users:read", Scope: "users:*"}}, + }, + userCache: &usertest.FakeUserService{ + GetSignedInUserFn: func(ctx context.Context, query *user.GetSignedInUserQuery) (*user.SignedInUser, error) { + return &user.SignedInUser{UserID: 1, OrgID: -1, Permissions: map[int64]map[string][]string{}}, nil + }, + }, + ctxSignedInUser: &user.SignedInUser{UserID: 1, OrgID: 1, Permissions: map[int64]map[string][]string{1: {"users:write": {"users:*"}}}}, + expectedStatus: http.StatusOK, + }, } // Run test cases diff --git a/pkg/services/accesscontrol/middleware.go b/pkg/services/accesscontrol/middleware.go index 6c76060542d..5ef36a8a216 100644 --- a/pkg/services/accesscontrol/middleware.go +++ b/pkg/services/accesscontrol/middleware.go @@ -260,23 +260,32 @@ func makeTmpUser(ctx context.Context, service Service, cache userCache, tmpUser.OrgName = queryResult.OrgName tmpUser.OrgRole = queryResult.OrgRole - if teamService != nil { - teamIDs, err := teamService.GetTeamIDsByUser(ctx, &team.GetTeamIDsByUserQuery{OrgID: targetOrgID, UserID: tmpUser.UserID}) - if err != nil { - return nil, err + // Only fetch the team membership is the user is a member of the organization + if queryResult.OrgID == targetOrgID { + if teamService != nil { + teamIDs, err := teamService.GetTeamIDsByUser(ctx, &team.GetTeamIDsByUserQuery{OrgID: targetOrgID, UserID: tmpUser.UserID}) + if err != nil { + return nil, err + } + tmpUser.Teams = teamIDs } - tmpUser.Teams = teamIDs } } } - if tmpUser.Permissions[targetOrgID] == nil || len(tmpUser.Permissions[targetOrgID]) == 0 { + // If the user is not a member of the organization + // evaluation must happen based on global permissions. + evaluationOrg := targetOrgID + if tmpUser.OrgID == NoOrgID { + evaluationOrg = GlobalOrgID + } + if tmpUser.Permissions[evaluationOrg] == nil || len(tmpUser.Permissions[evaluationOrg]) == 0 { permissions, err := service.GetUserPermissions(ctx, tmpUser, Options{}) if err != nil { return nil, err } - tmpUser.Permissions[targetOrgID] = GroupScopesByAction(permissions) + tmpUser.Permissions[evaluationOrg] = GroupScopesByAction(permissions) } return tmpUser, nil diff --git a/pkg/services/accesscontrol/models.go b/pkg/services/accesscontrol/models.go index ce353e721c5..68af6d37ddb 100644 --- a/pkg/services/accesscontrol/models.go +++ b/pkg/services/accesscontrol/models.go @@ -330,6 +330,7 @@ func (cmd *SaveExternalServiceRoleCommand) Validate() error { const ( GlobalOrgID = 0 + NoOrgID = int64(-1) GeneralFolderUID = "general" RoleGrafanaAdmin = "Grafana Admin" diff --git a/pkg/services/auth/identity/requester.go b/pkg/services/auth/identity/requester.go index b88120e88f2..aa7670d6c79 100644 --- a/pkg/services/auth/identity/requester.go +++ b/pkg/services/auth/identity/requester.go @@ -40,6 +40,8 @@ type Requester interface { GetOrgRole() roletype.RoleType // GetPermissions returns the permissions of the active entity. GetPermissions() map[string][]string + // GetGlobalPermissions returns the permissions of the active entity that are available across all organizations. + GetGlobalPermissions() map[string][]string // DEPRECATED: GetTeams returns the teams the entity is a member of. // Retrieve the teams from the team service instead of using this method. GetTeams() []int64 diff --git a/pkg/services/authn/identity.go b/pkg/services/authn/identity.go index dc8f8cce04c..4f618cd4424 100644 --- a/pkg/services/authn/identity.go +++ b/pkg/services/authn/identity.go @@ -31,6 +31,7 @@ const ( const ( AnonymousNamespaceID = NamespaceAnonymous + ":0" + GlobalOrgID = int64(0) ) var _ identity.Requester = (*Identity)(nil) @@ -164,6 +165,19 @@ func (i *Identity) GetPermissions() map[string][]string { return i.Permissions[i.GetOrgID()] } +// GetGlobalPermissions returns the permissions of the active entity that are available across all organizations +func (i *Identity) GetGlobalPermissions() map[string][]string { + if i.Permissions == nil { + return make(map[string][]string) + } + + if i.Permissions[GlobalOrgID] == nil { + return make(map[string][]string) + } + + return i.Permissions[GlobalOrgID] +} + func (i *Identity) GetTeams() []int64 { return i.Teams } diff --git a/pkg/services/user/identity.go b/pkg/services/user/identity.go index 49b4edc2c8c..2ab4b1f3661 100644 --- a/pkg/services/user/identity.go +++ b/pkg/services/user/identity.go @@ -8,6 +8,10 @@ import ( "github.com/grafana/grafana/pkg/services/auth/identity" ) +const ( + GlobalOrgID = int64(0) +) + type SignedInUser struct { UserID int64 `xorm:"user_id"` OrgID int64 `xorm:"org_id"` @@ -159,6 +163,19 @@ func (u *SignedInUser) GetPermissions() map[string][]string { return u.Permissions[u.GetOrgID()] } +// GetGlobalPermissions returns the permissions of the active entity that are available across all organizations +func (u *SignedInUser) GetGlobalPermissions() map[string][]string { + if u.Permissions == nil { + return make(map[string][]string) + } + + if u.Permissions[GlobalOrgID] == nil { + return make(map[string][]string) + } + + return u.Permissions[GlobalOrgID] +} + // DEPRECATED: GetTeams returns the teams the entity is a member of // Retrieve the teams from the team service instead of using this method. func (u *SignedInUser) GetTeams() []int64 { diff --git a/pkg/tests/api/alerting/api_backtesting_test.go b/pkg/tests/api/alerting/api_backtesting_test.go index de83add0e0f..08c60978bb7 100644 --- a/pkg/tests/api/alerting/api_backtesting_test.go +++ b/pkg/tests/api/alerting/api_backtesting_test.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/models/roletype" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions" "github.com/grafana/grafana/pkg/services/datasources" @@ -92,9 +93,10 @@ func TestBacktesting(t *testing.T) { t.Run("if user does not have permissions", func(t *testing.T) { testUserId := createUser(t, env.SQLStore, user.CreateUserCommand{ - DefaultOrgRole: "", + DefaultOrgRole: string(roletype.RoleNone), Password: "test", Login: "test", + OrgID: 1, }) testUserApiCli := newAlertingApiClient(grafanaListedAddr, "test", "test") From bac4c7fb326e158e5ebc1bdbab86104f7dfb9bf7 Mon Sep 17 00:00:00 2001 From: brendamuir <100768211+brendamuir@users.noreply.github.com> Date: Thu, 1 Feb 2024 12:43:11 +0100 Subject: [PATCH 37/84] Alerting docs: corrects save text (#81700) * Alerting docs: corrects save text * updates numbering --- .../template-notifications/use-notification-templates.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/alerting/manage-notifications/template-notifications/use-notification-templates.md b/docs/sources/alerting/manage-notifications/template-notifications/use-notification-templates.md index 78f2293b433..123124bc443 100644 --- a/docs/sources/alerting/manage-notifications/template-notifications/use-notification-templates.md +++ b/docs/sources/alerting/manage-notifications/template-notifications/use-notification-templates.md @@ -26,13 +26,13 @@ In the Contact points tab, you can see a list of your contact points. **Note:** You can edit an existing contact by clicking the Edit icon. -2. Execute a template from one or more fields such as Message and Subject: +1. Execute a template from one or more fields such as Message and Subject: {{< figure max-width="940px" src="/static/img/docs/alerting/unified/use-notification-template-9-4.png" caption="Use notification template" >}} For more information on how to write and execute templates, refer to [Using Go's templating language][using-go-templating-language] and [Create notification templates][create-notification-templates]. -3. Click Save template. +1. Click **Save contact point**. {{% docs/reference %}} [create-notification-templates]: "/docs/grafana/ -> /docs/grafana//alerting/manage-notifications/template-notifications/create-notification-templates" From 967a650d2d1bfa7d1f09e1fe7c743785f4217d0e Mon Sep 17 00:00:00 2001 From: Laura Benz <48948963+L-M-K-B@users.noreply.github.com> Date: Thu, 1 Feb 2024 12:44:18 +0100 Subject: [PATCH 38/84] I18n: Refactor crowdin workflow (#81571) * refactor: crowdin config * feat: add GH actions * refactor: remove old GH action * refactor: fix formatting issue * refactor: adjust docs * refactor: add changes after code review * refactor: add changes after code review * refactor: update CODEOWNERS file --- .github/CODEOWNERS | 3 +- .github/workflows/i18n-crowdin-download.yml | 51 +++++++++++++++ .github/workflows/i18n-crowdin-fix-files.yml | 67 -------------------- .github/workflows/i18n-crowdin-upload.yml | 29 +++++++++ contribute/internationalization.md | 20 +++--- crowdin.yml | 7 +- public/locales/i18next-parser.config.js | 4 +- 7 files changed, 98 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/i18n-crowdin-download.yml delete mode 100644 .github/workflows/i18n-crowdin-fix-files.yml create mode 100644 .github/workflows/i18n-crowdin-upload.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 166112a5a7d..1ccee149cb5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -678,7 +678,8 @@ embed.go @grafana/grafana-as-code /.github/workflows/ephemeral-instances-pr-opened-closed.yml @grafana/grafana-operator-experience-squad /.github/workflows/create-security-patch-from-security-mirror.yml @grafana/grafana-release-guild /.github/workflows/core-plugins-build-and-release.yml @grafana/plugins-platform-frontend @grafana/plugins-platform-backend -/.github/workflows/i18n-crowdin-fix-files.yml @grafana/grafana-frontend-platform +/.github/workflows/i18n-crowdin-upload.yml @grafana/grafana-frontend-platform +/.github/workflows/i18n-crowdin-download.yml @grafana/grafana-frontend-platform /.github/workflows/feature-toggle-cleanup.yml @tolzhabayev /.github/workflows/scripts/feature-toggle-cleanup/feature-toggle-cleanup.js @tolzhabayev diff --git a/.github/workflows/i18n-crowdin-download.yml b/.github/workflows/i18n-crowdin-download.yml new file mode 100644 index 00000000000..89d8a1c303e --- /dev/null +++ b/.github/workflows/i18n-crowdin-download.yml @@ -0,0 +1,51 @@ +name: Crowdin Download Action + +on: + workflow_dispatch: + schedule: + - cron: "0 * * * *" + +jobs: + download-sources-from-crowdin: + runs-on: ubuntu-latest + + permissions: + contents: write # needed to commit changes into the PR + pull-requests: write # needed to update PR description, labels, etc + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Download sources + uses: crowdin/github-action@v1 + with: + upload_sources: false + upload_translations: false + download_sources: false + download_translations: true + localization_branch_name: i18n_crowdin_translations + create_pull_request: true + pull_request_title: 'I18n: Download translations from Crowdin' + pull_request_body: | + :robot: Automatic download of translations from Crowdin. + + Steps for merging: + 1. A quick sanity check of the changes and approve. Things to look out for: + - No changes in the English file. The source of truth is in the main branch, NOT in Crowdin. + - Translations maybe be removed if the English phrase was removed, but there should not be many of these + - Anything else that looks 'funky'. Ask if you're not sure. + 2. Approve & (Auto-)merge. :tada: + + If there's a conflict, close the pull request and **delete the branch**. A GH action will recreate the pull request. + Remember, the longer this pull request is open, the more likely it is that it'll get conflicts. + pull_request_labels: 'area/frontend, area/internationalization, no-changelog, no-backport' + pull_request_reviewers: 'grafana-frontend-platform' + pull_request_base_branch_name: 'main' + base_url: 'https://grafana.api.crowdin.com' + config: 'crowdin.yml' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} + CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} diff --git a/.github/workflows/i18n-crowdin-fix-files.yml b/.github/workflows/i18n-crowdin-fix-files.yml deleted file mode 100644 index 5117bd381d6..00000000000 --- a/.github/workflows/i18n-crowdin-fix-files.yml +++ /dev/null @@ -1,67 +0,0 @@ -# When Crowdin creates a pull request from the crowdin-service-branch branch, -# run `yarn i18n:extract` and commit the changed grafana.json files back into the PR -# to reformat crowdin's changes to prevent conflicts with our CI checks. - -name: Fix Crowdin I18n files - -on: - pull_request: - paths: - - 'public/locales/*/grafana.json' - branches: - - main # Only run on pull requests *target* main (will be merged into main) - -jobs: - fix-files: - # Only run on pull requests *from* the crowdin-service-branch branch - if: github.head_ref == 'crowdin-service-branch' - - name: Fix files - runs-on: ubuntu-latest - - - permissions: - contents: write # needed to commit changes back into the PR - pull-requests: write # needed to update PR description - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - - uses: actions/setup-node@v4 - with: - node-version: 20.9.0 - cache: 'yarn' - - - name: Install dependencies - run: yarn install - - - name: Extract I18n files - run: yarn i18n:extract - - - name: Commit changes - uses: stefanzweifel/git-auto-commit-action@8756aa072ef5b4a080af5dc8fef36c5d586e521d # v5.0.0 - with: - commit_message: "Github Action: Auto-fix i18n files" - file_pattern: public/locales/*/grafana.json - - - name: Update PR description - uses: devindford/Append_PR_Comment@32dd2619cd96ac8da9907c416c992fe265233ca8 # v1.1.3 - if: ${{ ! contains(github.event.pull_request.body, 'Steps for merging') }} - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - body-update-action: prefix - body-template: | - :robot: Automatic sync of translations from Crowdin. - - Steps for merging: - 1. Wait for the "Github Action: Auto-fix i18n files" commit that may be required for CI to pass. - 2. A quick sanity check of the changes and approve. Things to look out for: - - No changes to the English strings. The source of truth is already in the main branch, NOT Crowdin. - - Translations maybe be removed if the English phrase was removed, but there should not be many of these - - Anything else that looks 'funky'. Ask if you're not sure. - 3. Approve & (Auto-)merge. :tada: - - If there's a conflict, close the pull request and **delete the branch**. Crowdin will recreate the pull request eventually. - Remember, the longer this pull request is open, the more likely it is that it'll get conflicts. diff --git a/.github/workflows/i18n-crowdin-upload.yml b/.github/workflows/i18n-crowdin-upload.yml new file mode 100644 index 00000000000..9dcdd5a6b10 --- /dev/null +++ b/.github/workflows/i18n-crowdin-upload.yml @@ -0,0 +1,29 @@ +name: Crowdin Upload Action + +on: + push: + paths: + - 'public/locales/en-US/grafana.json' + branches: + - main + +jobs: + upload-sources-to-crowdin: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Upload sources + uses: crowdin/github-action@v1 + with: + upload_sources: true + upload_translations: false + download_translations: false + create_pull_request: false + base_url: 'https://grafana.api.crowdin.com' + config: 'crowdin.yml' + env: + CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} + CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} diff --git a/contribute/internationalization.md b/contribute/internationalization.md index 243db35efe2..8d32a65e3ed 100644 --- a/contribute/internationalization.md +++ b/contribute/internationalization.md @@ -9,7 +9,8 @@ Grafana uses the [i18next](https://www.i18next.com/) framework for managing tran - Use `Go to {{ pageTitle }}` in code to add a translatable phrase - Translations are stored in JSON files in `public/locales/{locale}/grafana.json` - If a particular phrase is not available in the a language then it will fall back to English -- To update phrases in English, edit the default phrase in both the component's source and the [English grafana.json message catalogue](../public/locales/en-US/grafana.json), then run `yarn i18n:extract`. +- To update phrases in English, edit the default phrase in the component's source and then run `yarn i18n:extract`. +- The single source of truth for en-US (fallback language) is in grafana/grafana, the single source of truth for any translated language is Crowdin - To update phrases in any translated language, edit the phrase in Crowdin. Do not edit the `{locale}/grafana.json` ## How to add a new translation phrase @@ -40,7 +41,8 @@ const ErrorMessage = ({ id, message }) => There 2. Upon reload, the default English phrase will appear on the page. -3. Before submitting your PR, run the `yarn i18n:extract` command to extract the messages you added into the `grafana.json` file and make them available for translation. +3. Before submitting your PR, run the `yarn i18n:extract` command to extract the messages you added into the `public/locales/en-US/grafana.json` file and make them available for translation. + **Note:** All other languages will receive their translations when they are ready to be downloaded from Crowdin. ### Plain JS usage @@ -64,20 +66,20 @@ While the `t` function can technically be used outside of React functions (e.g, ## How to add a new language -1. Add new locale in Crowdin and sync files to repo +1. Add new locale in Crowdin and download files to repo 1. Grafana OSS Crowdin project -> "dot dot dot" menu in top right -> Target languages - 2. Grafana OSS Crowdin project -> Integrations -> Github -> Sync Now - 3. If Crowdin's locale code is different from our IETF language tag, add a custom mapping in Project Settings -> Language mapping -2. Update `public/app/core/internationalization/constants.ts` (add new constant, and add to `LOCALES`) -3. Update `public/locales/i18next-parser.config.js` to add the new locale to `locales` -4. Run `yarn i18n:extract` and commit the result + 2. If Crowdin's locale code is different from our IETF language tag, add a custom mapping in Project Settings -> Language mapping + 3. GH repo grafana/grafana -> Actions -> Choose `Crowdin Download Action` -> Run workflow -> Creates a PR automatically +2. Review the PR `I18n: Download translations from Crowdin` +3. Update `public/app/core/internationalization/constants.ts` (add new constant, and add to `LOCALES`) and add changes to the open PR +4. Approve and merge the PR ## How translations work in Grafana Grafana uses the [i18next](https://www.i18next.com/) framework for managing translating phrases in the Grafana frontend. It: - Marks up phrases within our code for extraction -- Extracts phrases into messages catalogues for translating in external systems +- Extracts phrases into the default messages catalogue for translating in external systems - Manages the user's locale and putting the translated phrases in the UI Grafana will load the message catalogue JSON before the initial render. diff --git a/crowdin.yml b/crowdin.yml index 1cdf69254df..ba8846585f1 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -2,7 +2,6 @@ files: - source: /public/locales/en-US/grafana.json translation: /public/locales/%locale%/%original_file_name% type: i18next_json -pull_request_title: 'I18n: Crowdin sync' -pull_request_labels: - - area/internationalization - - no-changelog +# The following are pulled from env variables +project_id_env: CROWDIN_PROJECT_ID +api_token_env: CROWDIN_PERSONAL_TOKEN diff --git a/public/locales/i18next-parser.config.js b/public/locales/i18next-parser.config.js index 0628220503a..f3f5bd6c9bc 100644 --- a/public/locales/i18next-parser.config.js +++ b/public/locales/i18next-parser.config.js @@ -1,8 +1,8 @@ module.exports = { // Default namespace used in your i18next config defaultNamespace: 'grafana', - - locales: ['en-US', 'fr-FR', 'es-ES', "de-DE", "zh-Hans", 'pseudo-LOCALE'], + // Adds changes only to en-US when extracting keys, every other language is provided by Crowdin + locales: ['en-US'], output: './public/locales/$LOCALE/$NAMESPACE.json', From 85a745ca9df444689342061869ff661e86d92a58 Mon Sep 17 00:00:00 2001 From: Josh Hunt Date: Thu, 1 Feb 2024 12:15:32 +0000 Subject: [PATCH 39/84] I18n: Trigger crowdin upload (#81704) --- public/locales/en-US/grafana.json | 2 +- public/locales/pseudo-LOCALE/grafana.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index 698ce2fc983..3670f46bea8 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -1,5 +1,5 @@ { - "_comment": "This file is the source of truth for English strings. Edit this to change plurals and other phrases for the UI.", + "_comment": "This file is the source of truth for English strings. Edit this in the codebase to change plurals and other phrases for the UI.", "access-control": { "add-permission": { "role-label": "Role", diff --git a/public/locales/pseudo-LOCALE/grafana.json b/public/locales/pseudo-LOCALE/grafana.json index ca1b477a369..e3d7a2aec7a 100644 --- a/public/locales/pseudo-LOCALE/grafana.json +++ b/public/locales/pseudo-LOCALE/grafana.json @@ -1,5 +1,5 @@ { - "_comment": "Ŧĥįş ƒįľę įş ŧĥę şőūřčę őƒ ŧřūŧĥ ƒőř Ēʼnģľįşĥ şŧřįʼnģş. Ēđįŧ ŧĥįş ŧő čĥäʼnģę pľūřäľş äʼnđ őŧĥęř pĥřäşęş ƒőř ŧĥę ŮĨ.", + "_comment": "Ŧĥįş ƒįľę įş ŧĥę şőūřčę őƒ ŧřūŧĥ ƒőř Ēʼnģľįşĥ şŧřįʼnģş. Ēđįŧ ŧĥįş įʼn ŧĥę čőđęþäşę ŧő čĥäʼnģę pľūřäľş äʼnđ őŧĥęř pĥřäşęş ƒőř ŧĥę ŮĨ.", "access-control": { "add-permission": { "role-label": "Ŗőľę", From 1671b775465e33276827a7f8e9f6736cbb6e643c Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:22:03 +0100 Subject: [PATCH 40/84] Loki: Fix processing of all lines labels in getParserAndLabelKeys (#81483) * Loki: Fix processing of all lines labels in getParserAndLabelKeys * Refactor * Update comment --- .../datasource/loki/responseUtils.test.ts | 33 +++++++++++++++++++ .../plugins/datasource/loki/responseUtils.ts | 19 +++++++---- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/loki/responseUtils.test.ts b/public/app/plugins/datasource/loki/responseUtils.test.ts index e4363b8b137..f961007968a 100644 --- a/public/app/plugins/datasource/loki/responseUtils.test.ts +++ b/public/app/plugins/datasource/loki/responseUtils.test.ts @@ -66,6 +66,34 @@ const frameWithTypes: DataFrame = { ], }; +const frameWithMultipleLabels: DataFrame = { + length: 1, + fields: [ + { + name: 'Time', + config: {}, + type: FieldType.time, + values: [1, 2, 3], + }, + { + name: 'labels', + config: {}, + type: FieldType.other, + values: [ + { level: 'info', foo: 'bar' }, + { level: 'info', foo: 'baz', new: 'yes' }, + { level: 'error', foo: 'baz' }, + ], + }, + { + name: 'Line', + config: {}, + type: FieldType.string, + values: ['line1', 'line2', 'line3'], + }, + ], +}; + describe('dataFrameHasParsingError', () => { it('handles frame with parsing error', () => { const input = cloneDeep(frame); @@ -138,6 +166,11 @@ describe('extractLabelKeysFromDataFrame', () => { expect(extractLabelKeysFromDataFrame(input)).toEqual(['level']); }); + it('extracts label keys from all logs', () => { + const input = cloneDeep(frameWithMultipleLabels); + expect(extractLabelKeysFromDataFrame(input)).toEqual(['level', 'foo', 'new']); + }); + it('extracts indexed label keys', () => { const input = cloneDeep(frameWithTypes); expect(extractLabelKeysFromDataFrame(input)).toEqual(['level']); diff --git a/public/app/plugins/datasource/loki/responseUtils.ts b/public/app/plugins/datasource/loki/responseUtils.ts index 6d9c05dc3d4..fd18c8f6c88 100644 --- a/public/app/plugins/datasource/loki/responseUtils.ts +++ b/public/app/plugins/datasource/loki/responseUtils.ts @@ -1,6 +1,6 @@ import { DataFrame, FieldType, isValidGoDuration, Labels } from '@grafana/data'; -import { isBytesString } from './languageUtils'; +import { isBytesString, processLabels } from './languageUtils'; import { isLogLineJSON, isLogLineLogfmt, isLogLinePacked } from './lineParser'; import { LabelType } from './types'; @@ -54,19 +54,26 @@ export function extractLabelKeysFromDataFrame(frame: DataFrame, type: LabelType return []; } - // if there are no label types, only return indexed labels if requested + // if there are no label types and type is LabelType.Indexed return all label keys if (!labelTypeArray?.length) { if (type === LabelType.Indexed) { - return Object.keys(labelsArray[0]); + const { keys: labelKeys } = processLabels(labelsArray); + return labelKeys; } return []; } - const labelTypes = labelTypeArray[0]; + // If we have label types, we can return only label keys that match type + let labelsSet = new Set(); + for (let i = 0; i < labelsArray.length; i++) { + const labels = labelsArray[i]; + const labelsType = labelTypeArray[i]; - const allLabelKeys = Object.keys(labelsArray[0]).filter((k) => labelTypes[k] === type); + const allLabelKeys = Object.keys(labels).filter((key) => labelsType[key] === type); + labelsSet = new Set([...labelsSet, ...allLabelKeys]); + } - return allLabelKeys; + return Array.from(labelsSet); } export function extractUnwrapLabelKeysFromDataFrame(frame: DataFrame): string[] { From fe1ed307c06ae764c8a44c51a794b4b461129242 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:22:18 +0100 Subject: [PATCH 41/84] Loki: Use fixed components from `@grafana/experimental` 1.7.9 (#81636) * Loki: Use fixed components from experimental * Update * Remove downloads.html --- .../loki/querybuilder/components/LokiQueryBuilder.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx index 0261c8f7bdb..652227a29ba 100644 --- a/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx +++ b/public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx @@ -5,7 +5,7 @@ import { usePrevious } from 'react-use'; import { DataSourceApi, getDefaultTimeRange, LoadingState, PanelData, SelectableValue, TimeRange } from '@grafana/data'; import { EditorRow, - // LabelFilters, this is broken in @grafana/experimental so we need to use the one from prometheus + LabelFilters, OperationExplainedBox, OperationList, OperationListExplained, @@ -16,7 +16,6 @@ import { QueryBuilderOperation, } from '@grafana/experimental'; import { config } from '@grafana/runtime'; -import { LabelFilters } from 'app/plugins/datasource/prometheus/querybuilder/shared/LabelFilters'; import { testIds } from '../../components/LokiQueryEditor'; import { LokiDatasource } from '../../datasource'; From 2d60c0123b51401f781f6c553c711eb6727d6575 Mon Sep 17 00:00:00 2001 From: Fabrizio <135109076+fabrizio-grafana@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:27:29 +0100 Subject: [PATCH 42/84] Chore: Improve how we install Python (#81695) --- .drone.yml | 35 ++++++++++++----------------------- scripts/drone/steps/lib.star | 2 +- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/.drone.yml b/.drone.yml index 47ff54c004d..a7112bfabae 100644 --- a/.drone.yml +++ b/.drone.yml @@ -123,8 +123,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -226,8 +225,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -539,8 +537,7 @@ steps: image: golang:1.21.6-alpine name: wire-install - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1050,8 +1047,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1411,8 +1407,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1489,8 +1484,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1549,8 +1543,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -1813,8 +1806,7 @@ steps: image: golang:1.21.6-alpine name: wire-install - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -2730,8 +2722,7 @@ steps: image: golang:1.21.6-alpine name: compile-build-cmd - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -3005,8 +2996,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -3430,8 +3420,7 @@ steps: image: alpine:3.18.5 name: identify-runner - commands: - - apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 - /usr/bin/python + - apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python - yarn install --immutable depends_on: [] image: node:20.9.0-alpine @@ -4799,6 +4788,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 88526f5c0dd7e3e225914c39c331b827eb7846fd83be24d9e055c0aff72a7e67 +hmac: c04928dc37abd22bc2d48c971f4f1fdfd722d38629016139fd0304e2aef60213 ... diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index e3c153e31cd..218b25e6028 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -36,7 +36,7 @@ def yarn_install_step(): "image": images["node"], "commands": [ # Python is needed to build `esfx`, which is needed by `msagl` - "apk add --no-cache --virtual .gyp python3 make g++ && ln -sf /usr/bin/python3 /usr/bin/python", + "apk add --update g++ make python3 && ln -sf /usr/bin/python3 /usr/bin/python", "yarn install --immutable", ], "depends_on": [], From 9de5e9754c36fed87e9aa94a322891b82f5b9007 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:46:32 +0100 Subject: [PATCH 43/84] Loki: Add to `.eslintrc` list to restrict imports from core (#81710) Loki: Add to .eslintrc list to restrict imports from core --- .eslintrc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index 0eacc7f00fd..ffa3fb0e977 100644 --- a/.eslintrc +++ b/.eslintrc @@ -111,7 +111,9 @@ "public/app/plugins/datasource/parca/*.{ts,tsx}", "public/app/plugins/datasource/parca/**/*.{ts,tsx}", "public/app/plugins/datasource/tempo/*.{ts,tsx}", - "public/app/plugins/datasource/tempo/**/*.{ts,tsx}" + "public/app/plugins/datasource/tempo/**/*.{ts,tsx}", + "public/app/plugins/datasource/loki/*.{ts,tsx}", + "public/app/plugins/datasource/loki/**/*.{ts,tsx}" ], "settings": { "import/resolver": { From 874ce14f0c992bf1bf567a00de3256f111207591 Mon Sep 17 00:00:00 2001 From: kay delaney <45561153+kaydelaney@users.noreply.github.com> Date: Thu, 1 Feb 2024 12:48:07 +0000 Subject: [PATCH 44/84] Scenes/PanelEditor: Add support for overrides tab and searching (#81595) --- .../panel-edit/PanelOptionsPane.tsx | 54 ++++++++++++++----- .../PanelEditor/OptionsPaneOptions.tsx | 2 +- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/public/app/features/dashboard-scene/panel-edit/PanelOptionsPane.tsx b/public/app/features/dashboard-scene/panel-edit/PanelOptionsPane.tsx index 9e0e23c5cb5..cebcbd46f73 100644 --- a/public/app/features/dashboard-scene/panel-edit/PanelOptionsPane.tsx +++ b/public/app/features/dashboard-scene/panel-edit/PanelOptionsPane.tsx @@ -5,6 +5,7 @@ import { GrafanaTheme2 } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; import { SceneComponentProps, SceneObjectBase, SceneObjectState, sceneGraph } from '@grafana/scenes'; import { ButtonGroup, FilterInput, RadioButtonGroup, ToolbarButton, useStyles2 } from '@grafana/ui'; +import { OptionFilter, renderSearchHits } from 'app/features/dashboard/components/PanelEditor/OptionsPaneOptions'; import { getFieldOverrideCategories } from 'app/features/dashboard/components/PanelEditor/getFieldOverrideElements'; import { getPanelFrameCategory2 } from 'app/features/dashboard/components/PanelEditor/getPanelFrameOptions'; import { getVisualizationOptions2 } from 'app/features/dashboard/components/PanelEditor/getVisualizationOptions'; @@ -35,6 +36,7 @@ export class PanelOptionsPane extends SceneObjectBase { const [isVizPickerOpen, setVizPickerOpen] = useState(true); const [searchQuery, setSearchQuery] = useState(''); const panelFrameOptions = useMemo(() => getPanelFrameCategory2(panel), [panel]); + const [listMode, setListMode] = useState(OptionFilter.All); const visualizationOptions = useMemo(() => { const plugin = panel.getPlugin(); @@ -68,11 +70,34 @@ export class PanelOptionsPane extends SceneObjectBase { [searchQuery, panel, fieldConfig] ); - const mainBoxElements = [ - panelFrameOptions.render(), - ...(visualizationOptions?.map((v) => v.render()) ?? []), - ...justOverrides.map((v) => v.render()), - ]; + const isSearching = searchQuery.length > 0; + const mainBoxElements: React.ReactNode[] = []; + + if (isSearching) { + mainBoxElements.push( + renderSearchHits([panelFrameOptions, ...(visualizationOptions ?? [])], justOverrides, searchQuery) + ); + } else { + switch (listMode) { + case OptionFilter.All: + mainBoxElements.push(panelFrameOptions.render()); + + for (const item of visualizationOptions ?? []) { + mainBoxElements.push(item.render()); + } + + for (const item of justOverrides) { + mainBoxElements.push(item.render()); + } + break; + case OptionFilter.Overrides: + for (const item of justOverrides) { + mainBoxElements.push(item.render()); + } + default: + break; + } + } return (
@@ -97,14 +122,17 @@ export class PanelOptionsPane extends SceneObjectBase { placeholder="Search options" onChange={setSearchQuery} /> - + {!isSearching && ( + + )}
{mainBoxElements}
diff --git a/public/app/features/dashboard/components/PanelEditor/OptionsPaneOptions.tsx b/public/app/features/dashboard/components/PanelEditor/OptionsPaneOptions.tsx index a218e5bd76b..db9039bb24d 100644 --- a/public/app/features/dashboard/components/PanelEditor/OptionsPaneOptions.tsx +++ b/public/app/features/dashboard/components/PanelEditor/OptionsPaneOptions.tsx @@ -150,7 +150,7 @@ export enum OptionFilter { Recent = 'Recent', } -function renderSearchHits( +export function renderSearchHits( allOptions: OptionsPaneCategoryDescriptor[], overrides: OptionsPaneCategoryDescriptor[], searchQuery: string From d0ecf863df9894dd227a1173b08a82644e7d1391 Mon Sep 17 00:00:00 2001 From: Andre Pereira Date: Thu, 1 Feb 2024 13:01:24 +0000 Subject: [PATCH 45/84] Datatrails: Metric as breadcrumb and style improvements (#81661) * Fixed duplicate trails entry for first time users. Show header with description for first time users * Renamed trail to history in History component * Use metric name in breadcrumb * Style tweaks around search field in metric select * Address PR comments * prettier --- public/app/features/trails/DataTrail.tsx | 11 +++-- public/app/features/trails/DataTrailCard.tsx | 18 ++------ public/app/features/trails/DataTrailsApp.tsx | 8 +++- .../app/features/trails/DataTrailsHistory.tsx | 2 +- public/app/features/trails/DataTrailsHome.tsx | 9 ++-- .../app/features/trails/MetricSelectScene.tsx | 43 +++++++++++-------- public/app/features/trails/MetricsHeader.tsx | 10 +++++ public/app/features/trails/utils.ts | 14 +++++- 8 files changed, 67 insertions(+), 48 deletions(-) create mode 100644 public/app/features/trails/MetricsHeader.tsx diff --git a/public/app/features/trails/DataTrail.tsx b/public/app/features/trails/DataTrail.tsx index a3cb526c225..27f75ee6b01 100644 --- a/public/app/features/trails/DataTrail.tsx +++ b/public/app/features/trails/DataTrail.tsx @@ -27,8 +27,9 @@ import { DataTrailSettings } from './DataTrailSettings'; import { DataTrailHistory, DataTrailHistoryStep } from './DataTrailsHistory'; import { MetricScene } from './MetricScene'; import { MetricSelectScene } from './MetricSelectScene'; +import { MetricsHeader } from './MetricsHeader'; import { getTrailStore } from './TrailStore/TrailStore'; -import { MetricSelectedEvent, trailDS, LOGS_METRIC, VAR_DATASOURCE, VAR_FILTERS } from './shared'; +import { LOGS_METRIC, MetricSelectedEvent, trailDS, VAR_DATASOURCE, VAR_FILTERS } from './shared'; import { getUrlForTrail } from './utils'; export interface DataTrailState extends SceneObjectState { @@ -162,7 +163,7 @@ export class DataTrail extends SceneObjectBase { } } else if (values.metric === null) { stateUpdate.metric = undefined; - stateUpdate.topScene = new MetricSelectScene({ showHeading: true }); + stateUpdate.topScene = new MetricSelectScene({}); } this.setState(stateUpdate); @@ -171,9 +172,11 @@ export class DataTrail extends SceneObjectBase { static Component = ({ model }: SceneComponentProps) => { const { controls, topScene, history } = model.useState(); const styles = useStyles2(getStyles); + const showHeaderForFirstTimeUsers = getTrailStore().recent.length < 2; return (
+ {showHeaderForFirstTimeUsers && } {controls && (
@@ -192,7 +195,7 @@ function getTopSceneFor(metric?: string) { if (metric) { return new MetricScene({ metric: metric }); } else { - return new MetricSelectScene({ showHeading: true }); + return new MetricSelectScene({}); } } @@ -221,7 +224,7 @@ function getStyles(theme: GrafanaTheme2) { container: css({ flexGrow: 1, display: 'flex', - gap: theme.spacing(2), + gap: theme.spacing(1), minHeight: '100%', flexDirection: 'column', }), diff --git a/public/app/features/trails/DataTrailCard.tsx b/public/app/features/trails/DataTrailCard.tsx index 757e18696fa..7e526c9640d 100644 --- a/public/app/features/trails/DataTrailCard.tsx +++ b/public/app/features/trails/DataTrailCard.tsx @@ -3,11 +3,11 @@ import React from 'react'; import { dateTimeFormat, GrafanaTheme2 } from '@grafana/data'; import { AdHocFiltersVariable, sceneGraph } from '@grafana/scenes'; -import { useStyles2, Stack, Card, IconButton, Badge } from '@grafana/ui'; +import { Badge, Card, IconButton, Stack, useStyles2 } from '@grafana/ui'; import { DataTrail } from './DataTrail'; -import { LOGS_METRIC, VAR_FILTERS } from './shared'; -import { getDataSource, getDataSourceName } from './utils'; +import { VAR_FILTERS } from './shared'; +import { getDataSource, getDataSourceName, getMetricName } from './utils'; export interface Props { trail: DataTrail; @@ -63,18 +63,6 @@ export function DataTrailCard({ trail, onSelect, onDelete }: Props) { ); } -function getMetricName(metric?: string) { - if (!metric) { - return 'Select metric'; - } - - if (metric === LOGS_METRIC) { - return 'Logs'; - } - - return metric; -} - function getStyles(theme: GrafanaTheme2) { return { tag: css({ diff --git a/public/app/features/trails/DataTrailsApp.tsx b/public/app/features/trails/DataTrailsApp.tsx index c56c8adc17e..f8fe2d33b5b 100644 --- a/public/app/features/trails/DataTrailsApp.tsx +++ b/public/app/features/trails/DataTrailsApp.tsx @@ -11,7 +11,7 @@ import { Page } from 'app/core/components/Page/Page'; import { DataTrail } from './DataTrail'; import { DataTrailsHome } from './DataTrailsHome'; import { getTrailStore } from './TrailStore/TrailStore'; -import { getUrlForTrail, newMetricsTrail } from './utils'; +import { getMetricName, getUrlForTrail, newMetricsTrail } from './utils'; export interface DataTrailsAppState extends SceneObjectState { trail: DataTrail; @@ -49,7 +49,11 @@ export class DataTrailsApp extends SceneObjectBase { exact={true} path="/explore/metrics/trail" render={() => ( - +
diff --git a/public/app/features/trails/DataTrailsHistory.tsx b/public/app/features/trails/DataTrailsHistory.tsx index b5bdbb6eeae..34490bd2fb1 100644 --- a/public/app/features/trails/DataTrailsHistory.tsx +++ b/public/app/features/trails/DataTrailsHistory.tsx @@ -139,7 +139,7 @@ export class DataTrailHistory extends SceneObjectBase { return (
-
Trail
+
History
{steps.map((step, index) => ( model.renderStepTooltip(step)} key={index}> diff --git a/public/app/features/trails/MetricSelectScene.tsx b/public/app/features/trails/MetricSelectScene.tsx index 13fd520cbfd..eb8ce11c199 100644 --- a/public/app/features/trails/MetricSelectScene.tsx +++ b/public/app/features/trails/MetricSelectScene.tsx @@ -4,22 +4,22 @@ import React from 'react'; import { GrafanaTheme2 } from '@grafana/data'; import { - SceneObjectState, - SceneObjectBase, - SceneComponentProps, PanelBuilders, - SceneFlexItem, - SceneVariableSet, QueryVariable, - sceneGraph, - VariableDependencyConfig, - SceneCSSGridLayout, + SceneComponentProps, SceneCSSGridItem, + SceneCSSGridLayout, + SceneFlexItem, + sceneGraph, + SceneObjectBase, SceneObjectRef, + SceneObjectState, SceneQueryRunner, + SceneVariableSet, + VariableDependencyConfig, } from '@grafana/scenes'; import { VariableHide } from '@grafana/schema'; -import { Input, Text, useStyles2, InlineSwitch, Field, LoadingPlaceholder } from '@grafana/ui'; +import { Field, Icon, InlineSwitch, Input, LoadingPlaceholder, useStyles2 } from '@grafana/ui'; import { getAutoQueriesForMetric } from './AutomaticMetricQueries/AutoQueryEngine'; import { MetricCategoryCascader } from './MetricCategory/MetricCategoryCascader'; @@ -40,7 +40,6 @@ interface MetricPanel { export interface MetricSelectSceneState extends SceneObjectState { body: SceneCSSGridLayout; - showHeading?: boolean; searchQuery?: string; showPreviews?: boolean; prefixFilter?: string; @@ -274,8 +273,7 @@ export class MetricSelectScene extends SceneObjectBase { }; public static Component = ({ model }: SceneComponentProps) => { - const { showHeading, searchQuery, showPreviews, body, metricsAfterSearch, metricsAfterFilter, prefixFilter } = - model.useState(); + const { searchQuery, showPreviews, body, metricsAfterSearch, metricsAfterFilter, prefixFilter } = model.useState(); const { children } = body.useState(); const styles = useStyles2(getStyles); @@ -296,13 +294,15 @@ export class MetricSelectScene extends SceneObjectBase { return (
- {showHeading && ( -
- Select a metric -
- )}
- + + } + value={searchQuery} + onChange={model.onSearchChange} + /> +
@@ -393,19 +393,24 @@ function getStyles(theme: GrafanaTheme2) { flexGrow: 1, }), headingWrapper: css({ - marginTop: theme.spacing(1), + marginBottom: theme.spacing(0.5), }), header: css({ flexGrow: 0, display: 'flex', gap: theme.spacing(2), marginBottom: theme.spacing(1), + alignItems: 'flex-end', }), statusMessage: css({ fontStyle: 'italic', marginTop: theme.spacing(7), textAlign: 'center', }), + searchField: css({ + flexGrow: 1, + marginBottom: 0, + }), }; } diff --git a/public/app/features/trails/MetricsHeader.tsx b/public/app/features/trails/MetricsHeader.tsx new file mode 100644 index 00000000000..f40edb4be89 --- /dev/null +++ b/public/app/features/trails/MetricsHeader.tsx @@ -0,0 +1,10 @@ +import React from 'react'; + +import { Stack, Text } from '@grafana/ui'; + +export const MetricsHeader = () => ( + + Metrics + Explore your Prometheus-compatible metrics without writing a query + +); diff --git a/public/app/features/trails/utils.ts b/public/app/features/trails/utils.ts index df5f62c0b77..b80a42b9eb5 100644 --- a/public/app/features/trails/utils.ts +++ b/public/app/features/trails/utils.ts @@ -8,7 +8,7 @@ import { DataTrail } from './DataTrail'; import { DataTrailSettings } from './DataTrailSettings'; import { MetricScene } from './MetricScene'; import { getTrailStore } from './TrailStore/TrailStore'; -import { TRAILS_ROUTE, VAR_DATASOURCE_EXPR } from './shared'; +import { LOGS_METRIC, TRAILS_ROUTE, VAR_DATASOURCE_EXPR } from './shared'; export function getTrailFor(model: SceneObject): DataTrail { return sceneGraph.getAncestor(model, DataTrail); @@ -58,6 +58,18 @@ export function getDataSourceName(dataSourceUid: string) { return getDataSourceSrv().getInstanceSettings(dataSourceUid)?.name || dataSourceUid; } +export function getMetricName(metric?: string) { + if (!metric) { + return 'Select metric'; + } + + if (metric === LOGS_METRIC) { + return 'Logs'; + } + + return metric; +} + export function getDatasourceForNewTrail(): string | undefined { const prevTrail = getTrailStore().recent[0]; if (prevTrail) { From 3e01ba0f5762c01c3ea54a85e4a7cda4a2f50f0a Mon Sep 17 00:00:00 2001 From: Ashley Harrison Date: Thu, 1 Feb 2024 13:03:11 +0000 Subject: [PATCH 46/84] QueryEditor: remove `slateAutocomplete` toggle (#81696) remove toggle --- .../feature-toggles/index.md | 1 - .../src/types/featureToggles.gen.ts | 1 - .../src/slate-plugins/suggestions.test.tsx | 70 ++++++------------- .../src/slate-plugins/suggestions.tsx | 16 ++--- pkg/services/featuremgmt/registry.go | 9 --- pkg/services/featuremgmt/toggles_gen.csv | 1 - pkg/services/featuremgmt/toggles_gen.go | 4 -- 7 files changed, 27 insertions(+), 75 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 516cf303af2..22cccf9e608 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -57,7 +57,6 @@ Some features are enabled by default. You can disable these feature by setting t | `lokiQueryHints` | Enables query hints for Loki | Yes | | `alertingQueryOptimization` | Optimizes eligible queries in order to reduce load on datasources | | | `jitterAlertRules` | Distributes alert rule evaluations more evenly over time, by rule group | | -| `slateAutocomplete` | Adjusts the behaviour of the slate editor to properly handle autocomplete. Feature toggled for safety. | Yes | ## Preview feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index fab1a2a7a62..b77af68741e 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -176,6 +176,5 @@ export interface FeatureToggles { onPremToCloudMigrations?: boolean; alertingSaveStatePeriodic?: boolean; promQLScope?: boolean; - slateAutocomplete?: boolean; nodeGraphDotLayout?: boolean; } diff --git a/packages/grafana-ui/src/slate-plugins/suggestions.test.tsx b/packages/grafana-ui/src/slate-plugins/suggestions.test.tsx index 8ce6c713d94..039e9bcac08 100644 --- a/packages/grafana-ui/src/slate-plugins/suggestions.test.tsx +++ b/packages/grafana-ui/src/slate-plugins/suggestions.test.tsx @@ -2,52 +2,28 @@ import { getNumCharsToDelete } from './suggestions'; describe('suggestions', () => { describe('getNumCharsToDelete', () => { - describe('when slateAutocomplete is enabled', () => { - let originalBootData = window.grafanaBootData; - - // hacky way to enable the feature toggle for this test - beforeEach(() => { - if (originalBootData) { - window.grafanaBootData = { - ...originalBootData, - settings: { - ...originalBootData.settings, - featureToggles: { - ...originalBootData.settings.featureToggles, - slateAutocomplete: true, - }, - }, - }; - } - }); - - afterEach(() => { - window.grafanaBootData = originalBootData; - }); - - const splunkCleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%:\\]/g, '').trim(); - it.each([ - // | represents the caret position - ['$query0 ', '', '', false, 0, undefined, { forward: 0, backward: 0 }], // "|" --> "$query0 |" - ['$query0 ', '$que', '$que', false, 0, undefined, { forward: 0, backward: 4 }], // "$que|" --> "$query0 |" - ['$query0 ', '$q', '$que', false, 0, undefined, { forward: 2, backward: 2 }], // "$q|ue" --> "$query0 |" - ['$query0 ', '$que', '($que)', false, 0, splunkCleanText, { forward: 0, backward: 4 }], // "($que|)" --> "($query0 |)" - ['$query0 ', '$que', 'esarvotionUsagePercent=$que', false, 0, undefined, { forward: 0, backward: 4 }], // "esarvotionUsagePercent=$que|" --> "esarvotionUsagePercent=$query0 |" - ])( - 'should calculate the correct number of characters to delete forwards and backwards', - (suggestionText, typeaheadPrefix, typeaheadText, preserveSuffix, deleteBackwards, cleanText, expected) => { - expect( - getNumCharsToDelete( - suggestionText, - typeaheadPrefix, - typeaheadText, - preserveSuffix, - deleteBackwards, - cleanText - ) - ).toEqual(expected); - } - ); - }); + const splunkCleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%:\\]/g, '').trim(); + it.each([ + // | represents the caret position + ['$query0 ', '', '', false, 0, undefined, { forward: 0, backward: 0 }], // "|" --> "$query0 |" + ['$query0 ', '$que', '$que', false, 0, undefined, { forward: 0, backward: 4 }], // "$que|" --> "$query0 |" + ['$query0 ', '$q', '$que', false, 0, undefined, { forward: 2, backward: 2 }], // "$q|ue" --> "$query0 |" + ['$query0 ', '$que', '($que)', false, 0, splunkCleanText, { forward: 0, backward: 4 }], // "($que|)" --> "($query0 |)" + ['$query0 ', '$que', 'esarvotionUsagePercent=$que', false, 0, undefined, { forward: 0, backward: 4 }], // "esarvotionUsagePercent=$que|" --> "esarvotionUsagePercent=$query0 |" + ])( + 'should calculate the correct number of characters to delete forwards and backwards', + (suggestionText, typeaheadPrefix, typeaheadText, preserveSuffix, deleteBackwards, cleanText, expected) => { + expect( + getNumCharsToDelete( + suggestionText, + typeaheadPrefix, + typeaheadText, + preserveSuffix, + deleteBackwards, + cleanText + ) + ).toEqual(expected); + } + ); }); }); diff --git a/packages/grafana-ui/src/slate-plugins/suggestions.tsx b/packages/grafana-ui/src/slate-plugins/suggestions.tsx index 9ef22285578..917895c437f 100644 --- a/packages/grafana-ui/src/slate-plugins/suggestions.tsx +++ b/packages/grafana-ui/src/slate-plugins/suggestions.tsx @@ -360,18 +360,10 @@ export function getNumCharsToDelete( const text = cleanText ? cleanText(typeaheadText) : typeaheadText; const offset = typeaheadText.indexOf(typeaheadPrefix); - let forward: number; - - if (window.grafanaBootData?.settings.featureToggles['slateAutocomplete']) { - const suffixLength = - offset > -1 ? text.length - offset - typeaheadPrefix.length : text.length - typeaheadPrefix.length; - const midWord = Boolean((typeaheadPrefix && suffixLength > 0) || suggestionText === typeaheadText); - forward = midWord && !preserveSuffix ? suffixLength + offset : 0; - } else { - const suffixLength = text.length - typeaheadPrefix.length; - const midWord = typeaheadPrefix && ((suffixLength > 0 && offset > -1) || suggestionText === typeaheadText); - forward = midWord && !preserveSuffix ? suffixLength + offset : 0; - } + const suffixLength = + offset > -1 ? text.length - offset - typeaheadPrefix.length : text.length - typeaheadPrefix.length; + const midWord = Boolean((typeaheadPrefix && suffixLength > 0) || suggestionText === typeaheadText); + const forward = midWord && !preserveSuffix ? suffixLength + offset : 0; return { forward, diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 20559900542..93c1b83dccb 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1337,15 +1337,6 @@ var ( Owner: grafanaObservabilityMetricsSquad, Created: time.Date(2024, time.January, 29, 0, 0, 0, 0, time.UTC), }, - { - Name: "slateAutocomplete", - Description: "Adjusts the behaviour of the slate editor to properly handle autocomplete. Feature toggled for safety.", - Stage: FeatureStageGeneralAvailability, - Expression: "true", // enabled by default - FrontendOnly: true, - Owner: grafanaFrontendPlatformSquad, - Created: time.Date(2024, time.January, 29, 12, 0, 0, 0, time.UTC), - }, { Name: "nodeGraphDotLayout", Description: "Changed the layout algorithm for the node graph", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 694c8a3e491..5777ab5db8c 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -157,5 +157,4 @@ jitterAlertRulesWithinGroups,preview,@grafana/alerting-squad,2024-01-17,false,tr onPremToCloudMigrations,experimental,@grafana/grafana-operator-experience-squad,2024-01-22,false,false,false alertingSaveStatePeriodic,privatePreview,@grafana/alerting-squad,2024-01-22,false,false,false promQLScope,experimental,@grafana/observability-metrics,2024-01-29,false,false,false -slateAutocomplete,GA,@grafana/grafana-frontend-platform,2024-01-29,false,false,true nodeGraphDotLayout,experimental,@grafana/observability-traces-and-profiling,2024-01-02,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 640d999ac1a..0c7f0594ec6 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -639,10 +639,6 @@ const ( // In-development feature that will allow injection of labels into prometheus queries. FlagPromQLScope = "promQLScope" - // FlagSlateAutocomplete - // Adjusts the behaviour of the slate editor to properly handle autocomplete. Feature toggled for safety. - FlagSlateAutocomplete = "slateAutocomplete" - // FlagNodeGraphDotLayout // Changed the layout algorithm for the node graph FlagNodeGraphDotLayout = "nodeGraphDotLayout" From eb889c41ee2006f48bbf7dd6e71dbde181ec6037 Mon Sep 17 00:00:00 2001 From: Gilles De Mey Date: Thu, 1 Feb 2024 14:17:48 +0100 Subject: [PATCH 47/84] Alerting: Fix support check for export with modifications (#81602) --- .../__snapshots__/useAbilities.test.tsx.snap | 66 +++++++++++++++++++ .../unified/hooks/useAbilities.test.tsx | 18 ++++- .../alerting/unified/hooks/useAbilities.ts | 4 +- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/public/app/features/alerting/unified/hooks/__snapshots__/useAbilities.test.tsx.snap b/public/app/features/alerting/unified/hooks/__snapshots__/useAbilities.test.tsx.snap index 52ae3a31f20..2b0417df565 100644 --- a/public/app/features/alerting/unified/hooks/__snapshots__/useAbilities.test.tsx.snap +++ b/public/app/features/alerting/unified/hooks/__snapshots__/useAbilities.test.tsx.snap @@ -1,5 +1,71 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`AlertRule abilities should report no permissions while we are loading data for cloud rule 1`] = ` +{ + "delete-alert-rule": [ + false, + false, + ], + "duplicate-alert-rule": [ + false, + false, + ], + "explore-alert-rule": [ + true, + false, + ], + "modify-export-rule": [ + false, + false, + ], + "silence-alert-rule": [ + false, + false, + ], + "update-alert-rule": [ + false, + false, + ], + "view-alert-rule": [ + true, + false, + ], +} +`; + +exports[`AlertRule abilities should report that all actions are supported for a Grafana Managed alert rule 1`] = ` +{ + "delete-alert-rule": [ + true, + false, + ], + "duplicate-alert-rule": [ + true, + false, + ], + "explore-alert-rule": [ + true, + false, + ], + "modify-export-rule": [ + true, + false, + ], + "silence-alert-rule": [ + true, + false, + ], + "update-alert-rule": [ + true, + false, + ], + "view-alert-rule": [ + true, + false, + ], +} +`; + exports[`alertmanager abilities should report Create / Update / Delete actions aren't supported for external vanilla alertmanager 1`] = ` { "create-contact-point": [ diff --git a/public/app/features/alerting/unified/hooks/useAbilities.test.tsx b/public/app/features/alerting/unified/hooks/useAbilities.test.tsx index 8d3e2ba8a5b..b0f12275dd4 100644 --- a/public/app/features/alerting/unified/hooks/useAbilities.test.tsx +++ b/public/app/features/alerting/unified/hooks/useAbilities.test.tsx @@ -7,7 +7,7 @@ import { TestProvider } from 'test/helpers/TestProvider'; import { AlertManagerDataSourceJsonData, AlertManagerImplementation } from 'app/plugins/datasource/alertmanager/types'; import { AccessControlAction } from 'app/types'; -import { getGrafanaRule, grantUserPermissions, mockDataSource } from '../mocks'; +import { getCloudRule, getGrafanaRule, grantUserPermissions, mockDataSource } from '../mocks'; import { AlertmanagerProvider } from '../state/AlertmanagerContext'; import { setupDataSources } from '../testSetup/datasources'; import { DataSourceType, GRAFANA_RULES_SOURCE_NAME } from '../utils/datasource'; @@ -136,7 +136,7 @@ describe('alertmanager abilities', () => { }); }); -describe('rule permissions', () => { +describe('AlertRule abilities', () => { it('should report that all actions are supported for a Grafana Managed alert rule', async () => { const rule = getGrafanaRule(); @@ -149,9 +149,21 @@ describe('rule permissions', () => { expect(supported).toBe(true); } }); + + expect(abilities.result.current).toMatchSnapshot(); }); - it('should report the correct set of supported actions for an external rule with ruler API', async () => {}); + it('should report no permissions while we are loading data for cloud rule', async () => { + const rule = getCloudRule(); + + const abilities = renderHook(() => useAllAlertRuleAbilities(rule), { wrapper: TestProvider }); + + await waitFor(() => { + expect(abilities.result.current).not.toBeUndefined(); + }); + + expect(abilities.result.current).toMatchSnapshot(); + }); it('should not allow certain actions for provisioned rules', () => {}); diff --git a/public/app/features/alerting/unified/hooks/useAbilities.ts b/public/app/features/alerting/unified/hooks/useAbilities.ts index df4cf5e5eb7..9906553bf7b 100644 --- a/public/app/features/alerting/unified/hooks/useAbilities.ts +++ b/public/app/features/alerting/unified/hooks/useAbilities.ts @@ -146,11 +146,11 @@ export function useAllAlertRuleAbilities(rule: CombinedRule): Abilities Date: Thu, 1 Feb 2024 14:18:43 +0100 Subject: [PATCH 48/84] Alerting: Refactor `useExternalDataSourceAlertmanagers` (#81081) --- .../alerting/unified/api/alertingApi.ts | 8 +- .../alerting/unified/api/dataSourcesApi.ts | 15 ++ .../admin/ExternalAlertmanagerDataSources.tsx | 18 +- .../admin/ExternalAlertmanagers.tsx | 5 +- .../components/rules/RuleListErrors.tsx | 6 +- .../hooks/useExternalAMSelector.test.tsx | 196 +++++++----------- .../unified/hooks/useExternalAmSelector.ts | 152 +++++++++----- public/app/features/alerting/unified/mocks.ts | 13 -- .../features/alerting/unified/utils/config.ts | 5 +- .../alerting/unified/utils/datasource.ts | 25 ++- .../features/alerting/unified/utils/misc.ts | 6 +- 11 files changed, 226 insertions(+), 223 deletions(-) create mode 100644 public/app/features/alerting/unified/api/dataSourcesApi.ts diff --git a/public/app/features/alerting/unified/api/alertingApi.ts b/public/app/features/alerting/unified/api/alertingApi.ts index 2ec7fc1d5c6..d4f9118395e 100644 --- a/public/app/features/alerting/unified/api/alertingApi.ts +++ b/public/app/features/alerting/unified/api/alertingApi.ts @@ -27,6 +27,12 @@ export const backendSrvBaseQuery = (): BaseQueryFn => async ( export const alertingApi = createApi({ reducerPath: 'alertingApi', baseQuery: backendSrvBaseQuery(), - tagTypes: ['AlertmanagerChoice', 'AlertmanagerConfiguration', 'OnCallIntegrations', 'OrgMigrationState'], + tagTypes: [ + 'AlertmanagerChoice', + 'AlertmanagerConfiguration', + 'OnCallIntegrations', + 'OrgMigrationState', + 'DataSourceSettings', + ], endpoints: () => ({}), }); diff --git a/public/app/features/alerting/unified/api/dataSourcesApi.ts b/public/app/features/alerting/unified/api/dataSourcesApi.ts new file mode 100644 index 00000000000..94bb2de0937 --- /dev/null +++ b/public/app/features/alerting/unified/api/dataSourcesApi.ts @@ -0,0 +1,15 @@ +import { DataSourceJsonData, DataSourceSettings } from '@grafana/data'; + +import { alertingApi } from './alertingApi'; + +export const dataSourcesApi = alertingApi.injectEndpoints({ + endpoints: (build) => ({ + getAllDataSourceSettings: build.query>, void>({ + query: () => ({ url: 'api/datasources' }), + // we'll create individual cache entries for each datasource UID + providesTags: (result) => { + return result ? result.map(({ uid }) => ({ type: 'DataSourceSettings', id: uid })) : ['DataSourceSettings']; + }, + }), + }), +}); diff --git a/public/app/features/alerting/unified/components/admin/ExternalAlertmanagerDataSources.tsx b/public/app/features/alerting/unified/components/admin/ExternalAlertmanagerDataSources.tsx index d84f930d3cc..757346b7976 100644 --- a/public/app/features/alerting/unified/components/admin/ExternalAlertmanagerDataSources.tsx +++ b/public/app/features/alerting/unified/components/admin/ExternalAlertmanagerDataSources.tsx @@ -5,11 +5,11 @@ import React from 'react'; import { GrafanaTheme2 } from '@grafana/data'; import { Badge, CallToActionCard, Card, Icon, LinkButton, Tooltip, useStyles2 } from '@grafana/ui'; -import { ExternalDataSourceAM } from '../../hooks/useExternalAmSelector'; +import { ExternalAlertmanagerDataSourceWithStatus } from '../../hooks/useExternalAmSelector'; import { makeDataSourceLink } from '../../utils/misc'; export interface ExternalAlertManagerDataSourcesProps { - alertmanagers: ExternalDataSourceAM[]; + alertmanagers: ExternalAlertmanagerDataSourceWithStatus[]; inactive: boolean; } @@ -39,7 +39,7 @@ export function ExternalAlertmanagerDataSources({ alertmanagers, inactive }: Ext {alertmanagers.length > 0 && (
{alertmanagers.map((am) => ( - + ))}
)} @@ -48,20 +48,20 @@ export function ExternalAlertmanagerDataSources({ alertmanagers, inactive }: Ext } interface ExternalAMdataSourceCardProps { - alertmanager: ExternalDataSourceAM; + alertmanager: ExternalAlertmanagerDataSourceWithStatus; inactive: boolean; } export function ExternalAMdataSourceCard({ alertmanager, inactive }: ExternalAMdataSourceCardProps) { const styles = useStyles2(getStyles); - const { dataSource, status, statusInconclusive, url } = alertmanager; + const { dataSourceSettings, status } = alertmanager; return ( - {dataSource.name}{' '} - {statusInconclusive && ( + {dataSourceSettings.name}{' '} + {status === 'inconclusive' && ( @@ -90,9 +90,9 @@ export function ExternalAMdataSourceCard({ alertmanager, inactive }: ExternalAMd /> )} - {url} + {dataSourceSettings.url} - + Go to datasource diff --git a/public/app/features/alerting/unified/components/admin/ExternalAlertmanagers.tsx b/public/app/features/alerting/unified/components/admin/ExternalAlertmanagers.tsx index 1bfb98fe018..ddcf5b9176d 100644 --- a/public/app/features/alerting/unified/components/admin/ExternalAlertmanagers.tsx +++ b/public/app/features/alerting/unified/components/admin/ExternalAlertmanagers.tsx @@ -23,6 +23,9 @@ export const ExternalAlertmanagers = () => { const dispatch = useDispatch(); const externalDsAlertManagers = useExternalDataSourceAlertmanagers(); + const gmaHandlingAlertmanagers = externalDsAlertManagers.filter( + (settings) => settings.dataSourceSettings.jsonData.handleGrafanaManagedAlerts === true + ); const { useSaveExternalAlertmanagersConfigMutation, @@ -71,7 +74,7 @@ export const ExternalAlertmanagers = () => {
diff --git a/public/app/features/alerting/unified/components/rules/RuleListErrors.tsx b/public/app/features/alerting/unified/components/rules/RuleListErrors.tsx index c595014ac96..16da99ee314 100644 --- a/public/app/features/alerting/unified/components/rules/RuleListErrors.tsx +++ b/public/app/features/alerting/unified/components/rules/RuleListErrors.tsx @@ -53,7 +53,7 @@ export function RuleListErrors(): ReactElement { result.push( <> Failed to load the data source configuration for{' '} - + {dataSource.name} : {error.message || 'Unknown error.'} @@ -65,7 +65,7 @@ export function RuleListErrors(): ReactElement { result.push( <> Failed to load rules state from{' '} - + {dataSource.name} : {error.message || 'Unknown error.'} @@ -77,7 +77,7 @@ export function RuleListErrors(): ReactElement { result.push( <> Failed to load rules config from{' '} - + {dataSource.name} : {error.message || 'Unknown error.'} diff --git a/public/app/features/alerting/unified/hooks/useExternalAMSelector.test.tsx b/public/app/features/alerting/unified/hooks/useExternalAMSelector.test.tsx index f45b2bc0ffe..6feda94a91c 100644 --- a/public/app/features/alerting/unified/hooks/useExternalAMSelector.test.tsx +++ b/public/app/features/alerting/unified/hooks/useExternalAMSelector.test.tsx @@ -1,16 +1,15 @@ import { renderHook, waitFor } from '@testing-library/react'; -import { setupServer } from 'msw/node'; -import React from 'react'; -import { Provider } from 'react-redux'; +import { rest } from 'msw'; +import { SetupServer, setupServer } from 'msw/node'; +import { TestProvider } from 'test/helpers/TestProvider'; import 'whatwg-fetch'; -import { DataSourceJsonData, DataSourceSettings } from '@grafana/data'; -import { config, setBackendSrv } from '@grafana/runtime'; +import { DataSourceSettings } from '@grafana/data'; +import { setBackendSrv } from '@grafana/runtime'; import { backendSrv } from 'app/core/services/backend_srv'; import { AlertManagerDataSourceJsonData } from 'app/plugins/datasource/alertmanager/types'; -import { mockDataSource, mockDataSourcesStore, mockStore } from '../mocks'; import { mockAlertmanagersResponse } from '../mocks/alertmanagerApi'; import { useExternalDataSourceAlertmanagers } from './useExternalAmSelector'; @@ -31,46 +30,42 @@ afterAll(() => { }); describe('useExternalDataSourceAlertmanagers', () => { - it('Should merge data sources information from config and api responses', async () => { + it('Should get the correct data source settings', async () => { // Arrange - const { dsSettings, dsInstanceSettings } = setupAlertmanagerDataSource({ url: 'http://grafana.com' }); - - config.datasources = { - 'External Alertmanager': dsInstanceSettings, - }; - - const store = mockDataSourcesStore({ - dataSources: [dsSettings], - }); - + setupAlertmanagerDataSource(server, { url: 'http://grafana.com' }); mockAlertmanagersResponse(server, { data: { activeAlertManagers: [], droppedAlertManagers: [] } }); - const wrapper = ({ children }: React.PropsWithChildren<{}>) => {children}; - // Act - const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper }); + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper: TestProvider }); await waitFor(() => { // Assert const { current } = result; expect(current).toHaveLength(1); - expect(current[0].dataSource.uid).toBe('1'); - expect(current[0].url).toBe('http://grafana.com'); + expect(current[0].dataSourceSettings.uid).toBe('1'); + expect(current[0].dataSourceSettings.url).toBe('http://grafana.com'); + }); + }); + + it('Should have uninterested state if data source does not want alerts', async () => { + // Arrange + setupAlertmanagerDataSource(server, { url: 'http://grafana.com', jsonData: { handleGrafanaManagedAlerts: false } }); + mockAlertmanagersResponse(server, { data: { activeAlertManagers: [], droppedAlertManagers: [] } }); + + // Act + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper: TestProvider }); + await waitFor(() => { + // Assert + const { current } = result; + + expect(current).toHaveLength(1); + expect(current[0].status).toBe('uninterested'); }); }); it('Should have active state if available in the activeAlertManagers', async () => { // Arrange - const { dsSettings, dsInstanceSettings } = setupAlertmanagerDataSource({ url: 'http://grafana.com' }); - - config.datasources = { - 'External Alertmanager': dsInstanceSettings, - }; - - const store = mockStore((state) => { - state.dataSources.dataSources = [dsSettings]; - }); - + setupAlertmanagerDataSource(server, { url: 'http://grafana.com' }); mockAlertmanagersResponse(server, { data: { activeAlertManagers: [{ url: 'http://grafana.com/api/v2/alerts' }], @@ -78,32 +73,20 @@ describe('useExternalDataSourceAlertmanagers', () => { }, }); - const wrapper = ({ children }: React.PropsWithChildren<{}>) => {children}; - // Act - const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper }); + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper: TestProvider }); await waitFor(() => { // Assert const { current } = result; expect(current).toHaveLength(1); expect(current[0].status).toBe('active'); - expect(current[0].statusInconclusive).toBe(false); }); }); it('Should have dropped state if available in the droppedAlertManagers', async () => { // Arrange - const { dsSettings, dsInstanceSettings } = setupAlertmanagerDataSource({ url: 'http://grafana.com' }); - - config.datasources = { - 'External Alertmanager': dsInstanceSettings, - }; - - const store = mockStore((state) => { - state.dataSources.dataSources = [dsSettings]; - }); - + setupAlertmanagerDataSource(server, { url: 'http://grafana.com' }); mockAlertmanagersResponse(server, { data: { activeAlertManagers: [], @@ -111,10 +94,8 @@ describe('useExternalDataSourceAlertmanagers', () => { }, }); - const wrapper = ({ children }: React.PropsWithChildren<{}>) => {children}; - // Act - const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper }); + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper: TestProvider }); await waitFor(() => { // Assert @@ -122,22 +103,12 @@ describe('useExternalDataSourceAlertmanagers', () => { expect(current).toHaveLength(1); expect(current[0].status).toBe('dropped'); - expect(current[0].statusInconclusive).toBe(false); }); }); it('Should have pending state if not available neither in dropped nor in active alertManagers', async () => { // Arrange - const { dsSettings, dsInstanceSettings } = setupAlertmanagerDataSource(); - - config.datasources = { - 'External Alertmanager': dsInstanceSettings, - }; - - const store = mockStore((state) => { - state.dataSources.dataSources = [dsSettings]; - }); - + setupAlertmanagerDataSource(server); mockAlertmanagersResponse(server, { data: { activeAlertManagers: [], @@ -145,10 +116,8 @@ describe('useExternalDataSourceAlertmanagers', () => { }, }); - const wrapper = ({ children }: React.PropsWithChildren<{}>) => {children}; - // Act - const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper }); + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper: TestProvider }); await waitFor(() => { // Assert @@ -156,22 +125,12 @@ describe('useExternalDataSourceAlertmanagers', () => { expect(current).toHaveLength(1); expect(current[0].status).toBe('pending'); - expect(current[0].statusInconclusive).toBe(false); }); }); it('Should match Alertmanager url when datasource url does not have protocol specified', async () => { // Arrange - const { dsSettings, dsInstanceSettings } = setupAlertmanagerDataSource({ url: 'localhost:9093' }); - - config.datasources = { - 'External Alertmanager': dsInstanceSettings, - }; - - const store = mockStore((state) => { - state.dataSources.dataSources = [dsSettings]; - }); - + setupAlertmanagerDataSource(server, { url: 'localhost:9093' }); mockAlertmanagersResponse(server, { data: { activeAlertManagers: [{ url: 'http://localhost:9093/api/v2/alerts' }], @@ -179,10 +138,8 @@ describe('useExternalDataSourceAlertmanagers', () => { }, }); - const wrapper = ({ children }: React.PropsWithChildren<{}>) => {children}; - // Act - const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper }); + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { wrapper: TestProvider }); await waitFor(() => { // Assert @@ -190,11 +147,34 @@ describe('useExternalDataSourceAlertmanagers', () => { expect(current).toHaveLength(1); expect(current[0].status).toBe('active'); - expect(current[0].url).toBe('localhost:9093'); + expect(current[0].dataSourceSettings.url).toBe('localhost:9093'); }); }); - it('Should have inconclusive state when there are many Alertmanagers of the same URL', async () => { + it('Should have inconclusive state when there are many Alertmanagers of the same URL on both active and inactive', async () => { + // Arrange + mockAlertmanagersResponse(server, { + data: { + activeAlertManagers: [{ url: 'http://grafana.com/api/v2/alerts' }], + droppedAlertManagers: [{ url: 'http://grafana.com/api/v2/alerts' }], + }, + }); + + setupAlertmanagerDataSource(server, { url: 'http://grafana.com' }); + + // Act + const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { + wrapper: TestProvider, + }); + + await waitFor(() => { + // Assert + expect(result.current).toHaveLength(1); + expect(result.current[0].status).toBe('inconclusive'); + }); + }); + + it('Should have not have inconclusive state when all Alertmanagers of the same URL are active', async () => { // Arrange mockAlertmanagersResponse(server, { data: { @@ -203,72 +183,40 @@ describe('useExternalDataSourceAlertmanagers', () => { }, }); - const { dsSettings, dsInstanceSettings } = setupAlertmanagerDataSource({ url: 'http://grafana.com' }); - - config.datasources = { - 'External Alertmanager': dsInstanceSettings, - }; - - const store = mockStore((state) => { - state.dataSources.dataSources = [dsSettings]; - }); - - const wrapper = ({ children }: React.PropsWithChildren<{}>) => {children}; + setupAlertmanagerDataSource(server, { url: 'http://grafana.com' }); // Act const { result } = renderHook(() => useExternalDataSourceAlertmanagers(), { - wrapper, + wrapper: TestProvider, }); await waitFor(() => { // Assert expect(result.current).toHaveLength(1); expect(result.current[0].status).toBe('active'); - expect(result.current[0].statusInconclusive).toBe(true); }); }); }); -function setupAlertmanagerDataSource(partialDsSettings?: Partial>) { +function setupAlertmanagerDataSource( + server: SetupServer, + partialDsSettings?: Partial> +) { const dsCommonConfig = { uid: '1', name: 'External Alertmanager', type: 'alertmanager', - jsonData: { handleGrafanaManagedAlerts: true } as AlertManagerDataSourceJsonData, + jsonData: { handleGrafanaManagedAlerts: true }, }; - const dsInstanceSettings = mockDataSource(dsCommonConfig); - - const dsSettings = mockApiDataSource({ + const dsSettings = { ...dsCommonConfig, ...partialDsSettings, - }); - - return { dsSettings, dsInstanceSettings }; -} - -function mockApiDataSource(partial: Partial> = {}) { - const dsSettings: DataSourceSettings = { - uid: '1', - id: 1, - name: '', - url: '', - type: '', - access: '', - orgId: 1, - typeLogoUrl: '', - typeName: '', - user: '', - database: '', - basicAuth: false, - isDefault: false, - basicAuthUser: '', - jsonData: { handleGrafanaManagedAlerts: true } as AlertManagerDataSourceJsonData, - secureJsonFields: {}, - readOnly: false, - withCredentials: false, - ...partial, }; - return dsSettings; + server.use( + rest.get('/api/datasources', (_req, res, ctx) => { + return res(ctx.json([dsSettings])); + }) + ); } diff --git a/public/app/features/alerting/unified/hooks/useExternalAmSelector.ts b/public/app/features/alerting/unified/hooks/useExternalAmSelector.ts index 8ffe85bb8cb..8cbe7228698 100644 --- a/public/app/features/alerting/unified/hooks/useExternalAmSelector.ts +++ b/public/app/features/alerting/unified/hooks/useExternalAmSelector.ts @@ -1,76 +1,114 @@ -import { countBy, keyBy } from 'lodash'; -import { createSelector } from 'reselect'; - -import { DataSourceInstanceSettings, DataSourceJsonData, DataSourceSettings } from '@grafana/data'; -import { AlertManagerDataSourceJsonData } from 'app/plugins/datasource/alertmanager/types'; -import { StoreState, useSelector } from 'app/types'; +import { DataSourceSettings } from '@grafana/data'; +import { AlertManagerDataSourceJsonData, ExternalAlertmanagers } from 'app/plugins/datasource/alertmanager/types'; import { alertmanagerApi } from '../api/alertmanagerApi'; -import { getAlertManagerDataSources } from '../utils/datasource'; +import { dataSourcesApi } from '../api/dataSourcesApi'; +import { isAlertmanagerDataSource } from '../utils/datasource'; -export interface ExternalDataSourceAM { - dataSource: DataSourceInstanceSettings; - url?: string; - status: 'active' | 'pending' | 'dropped'; - statusInconclusive?: boolean; +type ConnectionStatus = 'active' | 'pending' | 'dropped' | 'inconclusive' | 'uninterested' | 'unknown'; + +export interface ExternalAlertmanagerDataSourceWithStatus { + dataSourceSettings: DataSourceSettings; + status: ConnectionStatus; } -export function useExternalDataSourceAlertmanagers(): ExternalDataSourceAM[] { - const { useGetExternalAlertmanagersQuery } = alertmanagerApi; - const { currentData: discoveredAlertmanagers } = useGetExternalAlertmanagersQuery(); +/** + * Returns all configured Alertmanager data sources and their connection status with the internal ruler + */ +export function useExternalDataSourceAlertmanagers(): ExternalAlertmanagerDataSourceWithStatus[] { + // firstly we'll fetch the settings for all datasources and filter for "alertmanager" type + const { alertmanagerDataSources } = dataSourcesApi.endpoints.getAllDataSourceSettings.useQuery(undefined, { + refetchOnReconnect: true, + selectFromResult: (result) => { + const alertmanagerDataSources = result.currentData?.filter(isAlertmanagerDataSource) ?? []; + return { ...result, alertmanagerDataSources }; + }, + }); - const externalDsAlertManagers = getAlertManagerDataSources().filter((ds) => ds.jsonData.handleGrafanaManagedAlerts); - - const alertmanagerDatasources = useSelector( - createSelector( - (state: StoreState) => state.dataSources.dataSources.filter((ds) => ds.type === 'alertmanager'), - (datasources) => keyBy(datasources, (ds) => ds.uid) - ) + // we'll also fetch the configuration for which Alertmanagers we are forwarding Grafana-managed alerts too + // @TODO use polling when we have one or more alertmanagers in pending state + const { currentData: externalAlertmanagers } = alertmanagerApi.endpoints.getExternalAlertmanagers.useQuery( + undefined, + { refetchOnReconnect: true } ); - const droppedAMUrls = countBy(discoveredAlertmanagers?.droppedAlertManagers, (x) => x.url); - const activeAMUrls = countBy(discoveredAlertmanagers?.activeAlertManagers, (x) => x.url); + if (!alertmanagerDataSources) { + return []; + } - return externalDsAlertManagers.map((dsAm) => { - const dsSettings = alertmanagerDatasources[dsAm.uid]; - - if (!dsSettings) { - return { - dataSource: dsAm, - status: 'pending', - }; - } - - const amUrl = getDataSourceUrlWithProtocol(dsSettings); - const amStatusUrl = `${amUrl}/api/v2/alerts`; - - const matchingDroppedUrls = droppedAMUrls[amStatusUrl] ?? 0; - const matchingActiveUrls = activeAMUrls[amStatusUrl] ?? 0; - - const isDropped = matchingDroppedUrls > 0; - const isActive = matchingActiveUrls > 0; - - // Multiple Alertmanagers of the same URL may exist (e.g. with different credentials) - // Alertmanager response only contains URLs, so in case of duplication, we are not able - // to distinguish which is which, resulting in an inconclusive status. - const isStatusInconclusive = matchingDroppedUrls + matchingActiveUrls > 1; - - const status = isDropped ? 'dropped' : isActive ? 'active' : 'pending'; + return alertmanagerDataSources.map((dataSourceSettings) => { + const status = externalAlertmanagers + ? determineAlertmanagerConnectionStatus(externalAlertmanagers, dataSourceSettings) + : 'unknown'; return { - dataSource: dsAm, - url: dsSettings.url, + dataSourceSettings, status, - statusInconclusive: isStatusInconclusive, }; }); } -function getDataSourceUrlWithProtocol(dsSettings: DataSourceSettings) { - const hasProtocol = new RegExp('^[^:]*://').test(dsSettings.url); - if (!hasProtocol) { - return `http://${dsSettings.url}`; // Grafana append http protocol if there is no any +// using the information from /api/v1/ngalert/alertmanagers we should derive the connection status of a single data source +function determineAlertmanagerConnectionStatus( + externalAlertmanagers: ExternalAlertmanagers, + dataSourceSettings: DataSourceSettings +): ConnectionStatus { + const isInterestedInAlerts = dataSourceSettings.jsonData.handleGrafanaManagedAlerts; + if (!isInterestedInAlerts) { + return 'uninterested'; } - return dsSettings.url; + const isActive = + externalAlertmanagers?.activeAlertManagers.some((am) => { + return isAlertmanagerMatchByURL(dataSourceSettings.url, am.url); + }) ?? []; + + const isDropped = + externalAlertmanagers?.droppedAlertManagers.some((am) => { + return isAlertmanagerMatchByURL(dataSourceSettings.url, am.url); + }) ?? []; + + // the Alertmanager is being adopted (pending) if it is interested in handling alerts but not in either "active" or "dropped" + const isPending = !isActive && !isDropped; + if (isPending) { + return 'pending'; + } + + // Multiple Alertmanagers of the same URL may exist (e.g. with different credentials) + // Alertmanager response only contains URLs, so when the URL exists in both active and dropped, we are not able + // to distinguish which is which, resulting in an inconclusive status. + const isInconclusive = isActive && isDropped; + if (isInconclusive) { + return 'inconclusive'; + } + + // if we get here, it's neither "uninterested", nor "inconclusive" nor "pending" + if (isActive) { + return 'active'; + } else if (isDropped) { + return 'dropped'; + } + + return 'unknown'; +} + +// the vanilla Alertmanager and Mimir Alertmanager mount their API endpoints on different sub-paths +// Cortex also uses the same paths as Mimir +const MIMIR_ALERTMANAGER_PATH = '/alertmanager/api/v2/alerts'; +const VANILLA_ALERTMANAGER_PATH = '/api/v2/alerts'; + +// when using the Mimir Alertmanager, those paths are mounted under "/alertmanager" +function isAlertmanagerMatchByURL(dataSourceUrl: string, alertmanagerUrl: string) { + const normalizedUrl = normalizeDataSourceURL(dataSourceUrl); + + const prometheusAlertmanagerMatch = alertmanagerUrl === `${normalizedUrl}${VANILLA_ALERTMANAGER_PATH}`; + const mimirAlertmanagerMatch = alertmanagerUrl === `${normalizedUrl}${MIMIR_ALERTMANAGER_PATH}`; + + return prometheusAlertmanagerMatch || mimirAlertmanagerMatch; +} + +// Grafana prepends the http protocol if there isn't one, but it doesn't store that in the datasource settings +function normalizeDataSourceURL(url: string) { + const hasProtocol = new RegExp('^[^:]*://').test(url); + return hasProtocol ? url : `http://${url}`; } diff --git a/public/app/features/alerting/unified/mocks.ts b/public/app/features/alerting/unified/mocks.ts index 1c6f623090f..88a216b58fb 100644 --- a/public/app/features/alerting/unified/mocks.ts +++ b/public/app/features/alerting/unified/mocks.ts @@ -602,19 +602,6 @@ export const grantUserPermissions = (permissions: AccessControlAction[]) => { .mockImplementation((action) => permissions.includes(action as AccessControlAction)); }; -export function mockDataSourcesStore(partial?: Partial) { - const defaultState = configureStore().getState(); - const store = configureStore({ - ...defaultState, - dataSources: { - ...defaultState.dataSources, - ...partial, - }, - }); - - return store; -} - export function mockUnifiedAlertingStore(unifiedAlerting?: Partial) { const defaultState = configureStore().getState(); diff --git a/public/app/features/alerting/unified/utils/config.ts b/public/app/features/alerting/unified/utils/config.ts index 9472a8b543e..d5f7817561d 100644 --- a/public/app/features/alerting/unified/utils/config.ts +++ b/public/app/features/alerting/unified/utils/config.ts @@ -1,12 +1,9 @@ import { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data'; import { config } from '@grafana/runtime'; -import { AlertManagerDataSourceJsonData } from 'app/plugins/datasource/alertmanager/types'; import { isValidPrometheusDuration, parsePrometheusDuration } from './time'; -export function getAllDataSources(): Array< - DataSourceInstanceSettings -> { +export function getAllDataSources(): Array> { return Object.values(config.datasources); } diff --git a/public/app/features/alerting/unified/utils/datasource.ts b/public/app/features/alerting/unified/utils/datasource.ts index de43e658649..e2f70cb8586 100644 --- a/public/app/features/alerting/unified/utils/datasource.ts +++ b/public/app/features/alerting/unified/utils/datasource.ts @@ -1,4 +1,4 @@ -import { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data'; +import { DataSourceInstanceSettings, DataSourceJsonData, DataSourceSettings } from '@grafana/data'; import { getDataSourceSrv } from '@grafana/runtime'; import { contextSrv } from 'app/core/services/context_srv'; import { @@ -51,12 +51,22 @@ export function getRulesDataSource(rulesSourceName: string) { export function getAlertManagerDataSources() { return getAllDataSources() - .filter( - (ds): ds is DataSourceInstanceSettings => ds.type === DataSourceType.Alertmanager - ) + .filter(isAlertmanagerDataSourceInstance) .sort((a, b) => a.name.localeCompare(b.name)); } +export function isAlertmanagerDataSourceInstance( + dataSource: DataSourceInstanceSettings +): dataSource is DataSourceInstanceSettings { + return dataSource.type === DataSourceType.Alertmanager; +} + +export function isAlertmanagerDataSource( + dataSource: DataSourceSettings +): dataSource is DataSourceSettings { + return dataSource.type === DataSourceType.Alertmanager; +} + export function getExternalDsAlertManagers() { return getAlertManagerDataSources().filter((ds) => ds.jsonData.handleGrafanaManagedAlerts); } @@ -205,10 +215,9 @@ export function getDataSourceByName(name: string): DataSourceInstanceSettings => - source.name === name && source.type === 'alertmanager' - ); + return getAllDataSources() + .filter(isAlertmanagerDataSourceInstance) + .find((source) => source.name === name); } export function getRulesSourceByName(name: string): RulesSource | undefined { diff --git a/public/app/features/alerting/unified/utils/misc.ts b/public/app/features/alerting/unified/utils/misc.ts index 1b0017f3601..c6e4739353d 100644 --- a/public/app/features/alerting/unified/utils/misc.ts +++ b/public/app/features/alerting/unified/utils/misc.ts @@ -1,6 +1,6 @@ import { sortBy } from 'lodash'; -import { UrlQueryMap, Labels, DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data'; +import { UrlQueryMap, Labels } from '@grafana/data'; import { GrafanaEdition } from '@grafana/data/src/types/config'; import { config } from '@grafana/runtime'; import { DataSourceRef } from '@grafana/schema'; @@ -137,8 +137,8 @@ export function makeLabelBasedSilenceLink(alertManagerSourceName: string, labels return createUrl('/alerting/silence/new', silenceUrlParams); } -export function makeDataSourceLink(dataSource: DataSourceInstanceSettings) { - return createUrl(`/datasources/edit/${dataSource.uid}`); +export function makeDataSourceLink(uid: string) { + return createUrl(`/datasources/edit/${uid}`); } export function makeFolderLink(folderUID: string): string { From 7c2622a4f1e325d50984498d2b55fd6afabfee71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 1 Feb 2024 15:07:01 +0100 Subject: [PATCH 49/84] ShareModal: Fixes url sync issue that caused issue with save drawer (#81706) * ShareModal: Fixes url sync issue that caused issue with save drawer * Updated fix --- .../components/DashNav/ShareButton.tsx | 29 +++---------------- .../components/ShareModal/ShareModal.tsx | 1 - .../dashboard/containers/DashboardPage.tsx | 6 ++++ .../features/dashboard/containers/types.ts | 1 + 4 files changed, 11 insertions(+), 26 deletions(-) diff --git a/public/app/features/dashboard/components/DashNav/ShareButton.tsx b/public/app/features/dashboard/components/DashNav/ShareButton.tsx index b1f3ca69dd7..2ff6ac53448 100644 --- a/public/app/features/dashboard/components/DashNav/ShareButton.tsx +++ b/public/app/features/dashboard/components/DashNav/ShareButton.tsx @@ -1,31 +1,13 @@ -import React, { useContext, useEffect } from 'react'; +import React from 'react'; import { selectors as e2eSelectors } from '@grafana/e2e-selectors/src'; -import { ModalsContext, Button } from '@grafana/ui'; -import { useQueryParams } from 'app/core/hooks/useQueryParams'; +import { locationService } from '@grafana/runtime'; +import { Button } from '@grafana/ui'; import { Trans } from 'app/core/internationalization'; import { DashboardModel } from 'app/features/dashboard/state'; import { DashboardInteractions } from 'app/features/dashboard-scene/utils/interactions'; -import { ShareModal } from '../ShareModal'; - export const ShareButton = ({ dashboard }: { dashboard: DashboardModel }) => { - const [queryParams] = useQueryParams(); - const { showModal, hideModal } = useContext(ModalsContext); - - useEffect(() => { - if (!!queryParams.shareView) { - showModal(ShareModal, { - dashboard, - onDismiss: hideModal, - activeTab: String(queryParams.shareView), - }); - } - return () => { - hideModal(); - }; - }, [showModal, hideModal, dashboard, queryParams.shareView]); - return ( )} -
diff --git a/public/app/features/dashboard-scene/settings/variables/VariableEditorList.tsx b/public/app/features/dashboard-scene/settings/variables/VariableEditorList.tsx index 8cbe42568ef..27d16634ae8 100644 --- a/public/app/features/dashboard-scene/settings/variables/VariableEditorList.tsx +++ b/public/app/features/dashboard-scene/settings/variables/VariableEditorList.tsx @@ -5,7 +5,7 @@ import { DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd'; import { selectors } from '@grafana/e2e-selectors'; import { reportInteraction } from '@grafana/runtime'; import { SceneVariable, SceneVariableState } from '@grafana/scenes'; -import { useStyles2, Stack } from '@grafana/ui'; +import { useStyles2, Stack, Button } from '@grafana/ui'; import EmptyListCTA from 'app/core/components/EmptyListCTA/EmptyListCTA'; import { VariableEditorListRow } from './VariableEditorListRow'; @@ -80,6 +80,15 @@ export function VariableEditorList({
+ + + )}
@@ -94,7 +103,6 @@ function EmptyVariablesList({ onAdd }: { onAdd: () => void }): ReactElement { title="There are no variables yet" buttonIcon="calculator-alt" buttonTitle="Add variable" - buttonDisabled infoBox={{ __html: `

Variables enable more interactive and dynamic dashboards. Instead of hard-coding things like server diff --git a/public/app/features/dashboard-scene/settings/variables/utils.test.ts b/public/app/features/dashboard-scene/settings/variables/utils.test.ts index 9cdafc13411..9725ee5fe5c 100644 --- a/public/app/features/dashboard-scene/settings/variables/utils.test.ts +++ b/public/app/features/dashboard-scene/settings/variables/utils.test.ts @@ -8,6 +8,7 @@ import { DataSourceVariable, AdHocFiltersVariable, TextBoxVariable, + SceneVariableSet, } from '@grafana/scenes'; import { DataQuery, DataSourceJsonData, VariableType } from '@grafana/schema'; import { SHARED_DASHBOARD_QUERY } from 'app/plugins/datasource/dashboard'; @@ -31,6 +32,8 @@ import { EditableVariableType, getDefinition, getOptionDataSourceTypes, + getNextAvailableId, + getVariableDefault, } from './utils'; const templateSrv = { @@ -249,3 +252,47 @@ describe('getOptionDataSourceTypes', () => { expect(optionTypes[1].label).toBe('ds1'); }); }); + +describe('getNextAvailableId', () => { + it('should return the initial ID for an empty array', () => { + const sceneVariables = new SceneVariableSet({ + variables: [], + }); + + expect(getNextAvailableId('query', sceneVariables.state.variables)).toBe('query0'); + }); + + it('should return a non-conflicting ID for a non-empty array', () => { + const variable = new QueryVariable({ + name: 'query0', + label: 'test-label', + description: 'test-desc', + value: ['selected-value'], + text: ['selected-value-text'], + datasource: { uid: 'fake-std', type: 'fake-std' }, + query: 'query', + includeAll: true, + allValue: 'test-all', + isMulti: true, + }); + + const sceneVariables = new SceneVariableSet({ + variables: [variable], + }); + + expect(getNextAvailableId('query', sceneVariables.state.variables)).toBe('query1'); + }); +}); + +describe('getVariableDefault', () => { + it('should return a QueryVariable instance with the correct name', () => { + const sceneVariables = new SceneVariableSet({ + variables: [], + }); + + const defaultVariable = getVariableDefault(sceneVariables.state.variables); + + expect(defaultVariable).toBeInstanceOf(QueryVariable); + expect(defaultVariable.state.name).toBe('query0'); + }); +}); diff --git a/public/app/features/dashboard-scene/settings/variables/utils.ts b/public/app/features/dashboard-scene/settings/variables/utils.ts index 4b1514a336e..8b6480b6c9e 100644 --- a/public/app/features/dashboard-scene/settings/variables/utils.ts +++ b/public/app/features/dashboard-scene/settings/variables/utils.ts @@ -12,6 +12,7 @@ import { AdHocFilterSet, SceneVariable, MultiValueVariable, + SceneVariableState, } from '@grafana/scenes'; import { VariableType } from '@grafana/schema'; @@ -122,6 +123,25 @@ export function getVariableScene(type: EditableVariableType, initialState: Commo } } +export function getVariableDefault(variables: Array>) { + const defaultVariableType = 'query'; + const nextVariableIdName = getNextAvailableId(defaultVariableType, variables); + return new QueryVariable({ + name: nextVariableIdName, + }); +} + +export function getNextAvailableId(type: VariableType, variables: Array>): string { + let counter = 0; + let nextId = `${type}${counter}`; + + while (variables.find((variable) => variable.state.name === nextId)) { + nextId = `${type}${++counter}`; + } + + return nextId; +} + export function hasVariableOptions(variable: SceneVariable): variable is MultiValueVariable { // variable options can be defined by state.options or state.intervals in case of interval variable return 'options' in variable.state || 'intervals' in variable.state; From 95f90127add6fe01795ff8166e52cfa600331cca Mon Sep 17 00:00:00 2001 From: brendamuir <100768211+brendamuir@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:50:29 +0100 Subject: [PATCH 62/84] Alerting docs: rename provisioning files (#81722) --- .../provision-alerting-resources/_index.md | 18 +++++++-------- .../file-provisioning/index.md | 22 +++++++++---------- .../terraform-provisioning/index.md | 4 ++-- .../view-provisioned-resources/index.md | 6 ++--- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/sources/alerting/set-up/provision-alerting-resources/_index.md b/docs/sources/alerting/set-up/provision-alerting-resources/_index.md index b64985632fd..63656ff6eb9 100644 --- a/docs/sources/alerting/set-up/provision-alerting-resources/_index.md +++ b/docs/sources/alerting/set-up/provision-alerting-resources/_index.md @@ -2,7 +2,7 @@ aliases: - ../provision-alerting-resources/ canonical: https://grafana.com/docs/grafana/latest/alerting/set-up/provision-alerting-resources/ -description: Provision alerting resources +description: Import and export alerting resources keywords: - grafana - alerting @@ -14,23 +14,23 @@ labels: - cloud - enterprise - oss -title: Provision Grafana Alerting resources +title: Import and export Grafana Alerting resources weight: 300 --- -# Provision Grafana Alerting resources +# Import and export Grafana Alerting resources -Alerting infrastructure is often complex, with many pieces of the pipeline that often live in different places. Scaling this across multiple teams and organizations is an especially challenging task. Grafana Alerting provisioning makes this process easier by enabling you to create, manage, and maintain your alerting data in a way that best suits your organization. +Alerting infrastructure is often complex, with many pieces of the pipeline that often live in different places. Scaling this across multiple teams and organizations is an especially challenging task. Importing and exporting (or provisioning) your alerting resources in Grafana Alerting makes this process easier by enabling you to create, manage, and maintain your alerting data in a way that best suits your organization. -Provisioning for Grafana Alerting supports alert rules, contact points, notification policies, mute timings, and templates. +You can import alert rules, contact points, notification policies, mute timings, and templates. -You cannot edit provisioned alerting resources in the Grafana UI in the same way as unprovisioned alerting resources. You can only edit provisioned contact points, notification policies, templates, and mute timings in the source where they were created. For example, if you provision your alerting resources using files from disk, you cannot edit the data in Terraform or from within Grafana. +You cannot edit imported alerting resources in the Grafana UI in the same way as alerting resources that were not imported. You can only edit imported contact points, notification policies, templates, and mute timings in the source where they were created. For example, if you manage your alerting resources using files from disk, you cannot edit the data in Terraform or from within Grafana. -To modify provisioned alert rules, you can use the **Modify export** feature to edit and then export. +To modify imported alert rules, you can use the **Modify export** feature to edit and then export. -Choose from the options below to provision your Grafana Alerting resources. +Choose from the options below to import your Grafana Alerting resources. -1. Use file provisioning to provision your Grafana Alerting resources, such as alert rules and contact points, through files on disk. +1. Use file provisioning to manage your Grafana Alerting resources, such as alert rules and contact points, through files on disk. {{% admonition type="note" %}} File provisioning is not available in Grafana Cloud instances. diff --git a/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md b/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md index 890633ea8e1..b99e6a70f22 100644 --- a/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md +++ b/docs/sources/alerting/set-up/provision-alerting-resources/file-provisioning/index.md @@ -14,13 +14,13 @@ labels: - cloud - enterprise - oss -title: Create and manage alerting resources using file provisioning +title: Use file provisioning to manage alerting resources weight: 100 --- -## Create and manage alerting resources using file provisioning +## Use file provisioning to manage alerting resources -Provision your alerting resources using files from disk. When you start Grafana, the data from these files is created in your Grafana system. Grafana adds any new resources you created, updates any that you changed, and deletes old ones. +Manage your alerting resources using files from disk. When you start Grafana, the data from these files is created in your Grafana system. Grafana adds any new resources you created, updates any that you changed, and deletes old ones. Arrange your files in a directory in a way that best suits your use case. For example, you can choose a team-based layout where every team has its own file, you can have one big file for all your teams; or you can have one file per resource type. @@ -28,9 +28,9 @@ Details on how to set up the files and which fields are required for each object **Note:** -Provisioning takes place during the initial set up of your Grafana system, but you can re-run it at any time using the [Grafana Admin API][reload-provisioning-configurations]. +Importing takes place during the initial set up of your Grafana system, but you can re-run it at any time using the [Grafana Admin API][reload-provisioning-configurations]. -### Provision alert rules +### Import alert rules Create or delete alert rules in your Grafana instance(s). @@ -41,11 +41,11 @@ Create or delete alert rules in your Grafana instance(s). Example configuration files can be found below. 1. Ensure that your files are in the right directory on the node running the Grafana server, so that they deploy alongside your Grafana instance(s). -1. Delete the alert rules in Grafana that will be provisioned. +1. Delete the alert rules in Grafana that are going to be imported. **Note:** - If you do not delete the alert rule, it will clash with the provisioned alert rule once uploaded. + If you do not delete the alert rule, it will clash with the imported alert rule once uploaded. Here is an example of a configuration file for creating alert rules. @@ -134,7 +134,7 @@ deleteRules: uid: my_id_1 ``` -### Provision contact points +### Import contact points Create or delete contact points in your Grafana instance(s). @@ -495,7 +495,7 @@ settings: {{ template "default.title" . }} ``` -### Provision notification policies +### Import notification policies Create or reset the notification policy tree in your Grafana instance(s). @@ -587,7 +587,7 @@ In Grafana, the entire notification policy tree is considered a single, large re Since the policy tree is a single resource, applying it will overwrite a policy tree created through any other means. -### Provision templates +### Import templates Create or delete templates in your Grafana instance(s). @@ -627,7 +627,7 @@ deleteTemplates: name: my_first_template ``` -### Provision mute timings +### Import mute timings Create or delete mute timings in your Grafana instance(s). diff --git a/docs/sources/alerting/set-up/provision-alerting-resources/terraform-provisioning/index.md b/docs/sources/alerting/set-up/provision-alerting-resources/terraform-provisioning/index.md index 00da7920444..0ad04166a99 100644 --- a/docs/sources/alerting/set-up/provision-alerting-resources/terraform-provisioning/index.md +++ b/docs/sources/alerting/set-up/provision-alerting-resources/terraform-provisioning/index.md @@ -13,11 +13,11 @@ labels: products: - enterprise - oss -title: Create and manage alerting resources using Terraform +title: Use Terraform to manage alerting resources weight: 200 --- -# Create and manage alerting resources using Terraform +# Use Terraform to manage alerting resources Use Terraform’s Grafana Provider to manage your alerting resources and provision them into your Grafana system. Terraform provider support for Grafana Alerting makes it easy to create, manage, and maintain your entire Grafana Alerting stack as code. diff --git a/docs/sources/alerting/set-up/provision-alerting-resources/view-provisioned-resources/index.md b/docs/sources/alerting/set-up/provision-alerting-resources/view-provisioned-resources/index.md index 96e8c1d4245..5b473a7776d 100644 --- a/docs/sources/alerting/set-up/provision-alerting-resources/view-provisioned-resources/index.md +++ b/docs/sources/alerting/set-up/provision-alerting-resources/view-provisioned-resources/index.md @@ -13,12 +13,12 @@ labels: - cloud - enterprise - oss -menuTitle: Manage provisioned resources in Grafana -title: Manage provisioned alerting resources in Grafana +menuTitle: Manage provisioned alerting resources +title: Manage provisioned alerting resources weight: 300 --- -# Manage provisioned alerting resources in Grafana +# Manage provisioned alerting resources Verify that your alerting resources were created in Grafana, as well as edit or export your provisioned alerting resources. From 8fc8b03d1daa2cc84e82b315744ec85ec483b6ab Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 1 Feb 2024 18:07:32 +0100 Subject: [PATCH 63/84] Prometheus: Remove featuremgmt imports (#81586) * remove featuremgmt imports * pass down feature flags instead of reading them for every query execution --- .../plugins_integration_test.go | 2 +- pkg/tsdb/prometheus/healthcheck_test.go | 8 ++--- pkg/tsdb/prometheus/heuristics_test.go | 6 ++-- pkg/tsdb/prometheus/prometheus.go | 18 +++++----- pkg/tsdb/prometheus/prometheus_test.go | 6 ++-- pkg/tsdb/prometheus/querydata/request.go | 34 +++++++++---------- pkg/tsdb/prometheus/querydata/request_test.go | 4 +-- pkg/tsdb/prometheus/querydata/response.go | 6 ++-- .../prometheus/querydata/response_test.go | 10 +++--- 9 files changed, 44 insertions(+), 50 deletions(-) diff --git a/pkg/services/pluginsintegration/plugins_integration_test.go b/pkg/services/pluginsintegration/plugins_integration_test.go index cfe8b05a5e3..ad03c22c67a 100644 --- a/pkg/services/pluginsintegration/plugins_integration_test.go +++ b/pkg/services/pluginsintegration/plugins_integration_test.go @@ -78,7 +78,7 @@ func TestIntegrationPluginManager(t *testing.T) { idb := influxdb.ProvideService(hcp, features) lk := loki.ProvideService(hcp, features, tracer) otsdb := opentsdb.ProvideService(hcp) - pr := prometheus.ProvideService(hcp, cfg, features) + pr := prometheus.ProvideService(hcp, cfg) tmpo := tempo.ProvideService(hcp) td := testdatasource.ProvideService() pg := postgres.ProvideService(cfg) diff --git a/pkg/tsdb/prometheus/healthcheck_test.go b/pkg/tsdb/prometheus/healthcheck_test.go index eaf38c317e7..88d55c25ca3 100644 --- a/pkg/tsdb/prometheus/healthcheck_test.go +++ b/pkg/tsdb/prometheus/healthcheck_test.go @@ -11,9 +11,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" + + "github.com/grafana/grafana/pkg/setting" ) type healthCheckProvider[T http.RoundTripper] struct { @@ -84,7 +84,7 @@ func Test_healthcheck(t *testing.T) { t.Run("should do a successful health check", func(t *testing.T) { httpProvider := getMockProvider[*healthCheckSuccessRoundTripper]() s := &Service{ - im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, &featuremgmt.FeatureManager{}, backend.NewLoggerWith("logger", "test"))), + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, backend.NewLoggerWith("logger", "test"))), } req := &backend.CheckHealthRequest{ @@ -100,7 +100,7 @@ func Test_healthcheck(t *testing.T) { t.Run("should return an error for an unsuccessful health check", func(t *testing.T) { httpProvider := getMockProvider[*healthCheckFailRoundTripper]() s := &Service{ - im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, &featuremgmt.FeatureManager{}, backend.NewLoggerWith("logger", "test"))), + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, backend.NewLoggerWith("logger", "test"))), } req := &backend.CheckHealthRequest{ diff --git a/pkg/tsdb/prometheus/heuristics_test.go b/pkg/tsdb/prometheus/heuristics_test.go index 70de2738dfc..89f1d5b9a88 100644 --- a/pkg/tsdb/prometheus/heuristics_test.go +++ b/pkg/tsdb/prometheus/heuristics_test.go @@ -14,7 +14,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" - "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" ) @@ -52,7 +52,7 @@ func Test_GetHeuristics(t *testing.T) { //httpProvider := getHeuristicsMockProvider(&rt) httpProvider := newHeuristicsSDKProvider(rt) s := &Service{ - im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, &featuremgmt.FeatureManager{}, backend.NewLoggerWith("logger", "test"))), + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, backend.NewLoggerWith("logger", "test"))), } req := HeuristicsRequest{ @@ -72,7 +72,7 @@ func Test_GetHeuristics(t *testing.T) { } httpProvider := newHeuristicsSDKProvider(rt) s := &Service{ - im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, &featuremgmt.FeatureManager{}, backend.NewLoggerWith("logger", "test"))), + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, backend.NewLoggerWith("logger", "test"))), } req := HeuristicsRequest{ diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 9c1a0db794a..ab91aaf7394 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -15,7 +15,7 @@ import ( apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" - "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/prometheus/client" "github.com/grafana/grafana/pkg/tsdb/prometheus/instrumentation" @@ -24,9 +24,8 @@ import ( ) type Service struct { - im instancemgmt.InstanceManager - features featuremgmt.FeatureToggles - logger log.Logger + im instancemgmt.InstanceManager + logger log.Logger } type instance struct { @@ -35,17 +34,16 @@ type instance struct { versionCache *cache.Cache } -func ProvideService(httpClientProvider *httpclient.Provider, cfg *setting.Cfg, features featuremgmt.FeatureToggles) *Service { +func ProvideService(httpClientProvider *httpclient.Provider, cfg *setting.Cfg) *Service { plog := backend.NewLoggerWith("logger", "tsdb.prometheus") plog.Debug("Initializing") return &Service{ - im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider, cfg, features, plog)), - features: features, - logger: plog, + im: datasource.NewInstanceManager(newInstanceSettings(httpClientProvider, cfg, plog)), + logger: plog, } } -func newInstanceSettings(httpClientProvider *httpclient.Provider, cfg *setting.Cfg, features featuremgmt.FeatureToggles, log log.Logger) datasource.InstanceFactoryFunc { +func newInstanceSettings(httpClientProvider *httpclient.Provider, cfg *setting.Cfg, log log.Logger) datasource.InstanceFactoryFunc { return func(ctx context.Context, settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { // Creates a http roundTripper. opts, err := client.CreateTransportOptions(ctx, settings, cfg, log) @@ -58,7 +56,7 @@ func newInstanceSettings(httpClientProvider *httpclient.Provider, cfg *setting.C } // New version using custom client and better response parsing - qd, err := querydata.New(httpClient, features, settings, log) + qd, err := querydata.New(httpClient, settings, log) if err != nil { return nil, err } diff --git a/pkg/tsdb/prometheus/prometheus_test.go b/pkg/tsdb/prometheus/prometheus_test.go index 735ed25e38c..02307493664 100644 --- a/pkg/tsdb/prometheus/prometheus_test.go +++ b/pkg/tsdb/prometheus/prometheus_test.go @@ -10,9 +10,9 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/datasource" "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/setting" ) type fakeSender struct{} @@ -69,7 +69,7 @@ func TestService(t *testing.T) { f := &fakeHTTPClientProvider{} httpProvider := getMockPromTestSDKProvider(f) service := &Service{ - im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, &featuremgmt.FeatureManager{}, backend.NewLoggerWith("logger", "test"))), + im: datasource.NewInstanceManager(newInstanceSettings(httpProvider, &setting.Cfg{}, backend.NewLoggerWith("logger", "test"))), } req := &backend.CallResourceRequest{ diff --git a/pkg/tsdb/prometheus/querydata/request.go b/pkg/tsdb/prometheus/querydata/request.go index ceb5e3877d9..1282354207e 100644 --- a/pkg/tsdb/prometheus/querydata/request.go +++ b/pkg/tsdb/prometheus/querydata/request.go @@ -15,7 +15,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/log" "github.com/grafana/grafana-plugin-sdk-go/backend/tracing" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/tsdb/intervalv2" "github.com/grafana/grafana/pkg/tsdb/prometheus/client" "github.com/grafana/grafana/pkg/tsdb/prometheus/models" @@ -44,14 +43,11 @@ type QueryData struct { ID int64 URL string TimeInterval string - enableDataplane bool - enableScope bool exemplarSampler func() exemplar.Sampler } func New( httpClient *http.Client, - features featuremgmt.FeatureToggles, settings backend.DataSourceInstanceSettings, plog log.Logger, ) (*QueryData, error) { @@ -84,8 +80,6 @@ func New( ID: settings.ID, URL: settings.URL, exemplarSampler: exemplarSampler, - enableDataplane: features.IsEnabledGlobally(featuremgmt.FlagPrometheusDataplane), - enableScope: features.IsEnabledGlobally(featuremgmt.FlagPromQLScope), }, nil } @@ -95,13 +89,17 @@ func (s *QueryData) Execute(ctx context.Context, req *backend.QueryDataRequest) Responses: backend.Responses{}, } + cfg := backend.GrafanaConfigFromContext(ctx) + hasPromQLScopeFeatureFlag := cfg.FeatureToggles().IsEnabled("promQLScope") + hasPrometheusDataplaneFeatureFlag := cfg.FeatureToggles().IsEnabled("prometheusDataplane") + for _, q := range req.Queries { - query, err := models.Parse(q, s.TimeInterval, s.intervalCalculator, fromAlert, s.enableScope) + query, err := models.Parse(q, s.TimeInterval, s.intervalCalculator, fromAlert, hasPromQLScopeFeatureFlag) if err != nil { return &result, err } - r := s.fetch(ctx, s.client, query) + r := s.fetch(ctx, s.client, query, hasPrometheusDataplaneFeatureFlag) if r == nil { s.log.FromContext(ctx).Debug("Received nil response from runQuery", "query", query.Expr) continue @@ -112,7 +110,7 @@ func (s *QueryData) Execute(ctx context.Context, req *backend.QueryDataRequest) return &result, nil } -func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models.Query) *backend.DataResponse { +func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models.Query, enablePrometheusDataplane bool) *backend.DataResponse { traceCtx, end := s.trace(ctx, q) defer end() @@ -125,14 +123,14 @@ func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models. } if q.InstantQuery { - res := s.instantQuery(traceCtx, client, q) + res := s.instantQuery(traceCtx, client, q, enablePrometheusDataplane) dr.Error = res.Error dr.Frames = res.Frames dr.Status = res.Status } if q.RangeQuery { - res := s.rangeQuery(traceCtx, client, q) + res := s.rangeQuery(traceCtx, client, q, enablePrometheusDataplane) if res.Error != nil { if dr.Error == nil { dr.Error = res.Error @@ -147,7 +145,7 @@ func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models. } if q.ExemplarQuery { - res := s.exemplarQuery(traceCtx, client, q) + res := s.exemplarQuery(traceCtx, client, q, enablePrometheusDataplane) if res.Error != nil { // If exemplar query returns error, we want to only log it and // continue with other results processing @@ -159,7 +157,7 @@ func (s *QueryData) fetch(ctx context.Context, client *client.Client, q *models. return dr } -func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query) backend.DataResponse { +func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { res, err := c.QueryRange(ctx, q) if err != nil { return backend.DataResponse{ @@ -175,10 +173,10 @@ func (s *QueryData) rangeQuery(ctx context.Context, c *client.Client, q *models. } }() - return s.parseResponse(ctx, q, res) + return s.parseResponse(ctx, q, res, enablePrometheusDataplaneFlag) } -func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query) backend.DataResponse { +func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { res, err := c.QueryInstant(ctx, q) if err != nil { return backend.DataResponse{ @@ -201,10 +199,10 @@ func (s *QueryData) instantQuery(ctx context.Context, c *client.Client, q *model } }() - return s.parseResponse(ctx, q, res) + return s.parseResponse(ctx, q, res, enablePrometheusDataplaneFlag) } -func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query) backend.DataResponse { +func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *models.Query, enablePrometheusDataplaneFlag bool) backend.DataResponse { res, err := c.QueryExemplars(ctx, q) if err != nil { return backend.DataResponse{ @@ -218,7 +216,7 @@ func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *mode s.log.Warn("Failed to close response body", "error", err) } }() - return s.parseResponse(ctx, q, res) + return s.parseResponse(ctx, q, res, enablePrometheusDataplaneFlag) } func (s *QueryData) trace(ctx context.Context, q *models.Query) (context.Context, func()) { diff --git a/pkg/tsdb/prometheus/querydata/request_test.go b/pkg/tsdb/prometheus/querydata/request_test.go index 40d3d783b76..488d676f25e 100644 --- a/pkg/tsdb/prometheus/querydata/request_test.go +++ b/pkg/tsdb/prometheus/querydata/request_test.go @@ -15,7 +15,6 @@ import ( p "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery" "github.com/grafana/kindsys" @@ -442,7 +441,6 @@ func setup() (*testContext, error) { JSONData: json.RawMessage(`{"timeInterval": "15s"}`), } - features := featuremgmt.WithFeatures() opts, err := client.CreateTransportOptions(context.Background(), settings, &setting.Cfg{}, log.New()) if err != nil { return nil, err @@ -453,7 +451,7 @@ func setup() (*testContext, error) { return nil, err } - queryData, _ := querydata.New(httpClient, features, settings, log.New()) + queryData, _ := querydata.New(httpClient, settings, log.New()) return &testContext{ httpProvider: httpProvider, diff --git a/pkg/tsdb/prometheus/querydata/response.go b/pkg/tsdb/prometheus/querydata/response.go index be09ff3e575..dd0e5f3f31b 100644 --- a/pkg/tsdb/prometheus/querydata/response.go +++ b/pkg/tsdb/prometheus/querydata/response.go @@ -18,7 +18,7 @@ import ( "github.com/grafana/grafana/pkg/util/converter" ) -func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *http.Response) backend.DataResponse { +func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *http.Response, enablePrometheusDataplaneFlag bool) backend.DataResponse { defer func() { if err := res.Body.Close(); err != nil { s.log.FromContext(ctx).Error("Failed to close response body", "err", err) @@ -30,7 +30,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt iter := jsoniter.Parse(jsoniter.ConfigDefault, res.Body, 1024) r := converter.ReadPrometheusStyleResult(iter, converter.Options{ - Dataplane: s.enableDataplane, + Dataplane: enablePrometheusDataplaneFlag, }) r.Status = backend.Status(res.StatusCode) @@ -41,7 +41,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt // The ExecutedQueryString can be viewed in QueryInspector in UI for i, frame := range r.Frames { - addMetadataToMultiFrame(q, frame, s.enableDataplane) + addMetadataToMultiFrame(q, frame, enablePrometheusDataplaneFlag) if i == 0 { frame.Meta.ExecutedQueryString = executedQueryString(q) } diff --git a/pkg/tsdb/prometheus/querydata/response_test.go b/pkg/tsdb/prometheus/querydata/response_test.go index e449f3610d8..62d27222d8e 100644 --- a/pkg/tsdb/prometheus/querydata/response_test.go +++ b/pkg/tsdb/prometheus/querydata/response_test.go @@ -19,7 +19,7 @@ func TestQueryData_parseResponse(t *testing.T) { t.Run("resultType is before result the field must parsed normally", func(t *testing.T) { resBody := `{"data":{"resultType":"vector", "result":[{"metric":{"__name__":"some_name","environment":"some_env","id":"some_id","instance":"some_instance:1234","job":"some_job","name":"another_name","region":"some_region"},"value":[1.1,"2"]}]},"status":"success"}` res := &http.Response{Body: io.NopCloser(bytes.NewBufferString(resBody))} - result := qd.parseResponse(context.Background(), &models.Query{}, res) + result := qd.parseResponse(context.Background(), &models.Query{}, res, false) assert.Nil(t, result.Error) assert.Len(t, result.Frames, 1) }) @@ -27,7 +27,7 @@ func TestQueryData_parseResponse(t *testing.T) { t.Run("resultType is after the result field must parsed normally", func(t *testing.T) { resBody := `{"data":{"result":[{"metric":{"__name__":"some_name","environment":"some_env","id":"some_id","instance":"some_instance:1234","job":"some_job","name":"another_name","region":"some_region"},"value":[1.1,"2"]}],"resultType":"vector"},"status":"success"}` res := &http.Response{Body: io.NopCloser(bytes.NewBufferString(resBody))} - result := qd.parseResponse(context.Background(), &models.Query{}, res) + result := qd.parseResponse(context.Background(), &models.Query{}, res, false) assert.Nil(t, result.Error) assert.Len(t, result.Frames, 1) }) @@ -35,7 +35,7 @@ func TestQueryData_parseResponse(t *testing.T) { t.Run("no resultType is existed in the data", func(t *testing.T) { resBody := `{"data":{"result":[{"metric":{"__name__":"some_name","environment":"some_env","id":"some_id","instance":"some_instance:1234","job":"some_job","name":"another_name","region":"some_region"},"value":[1.1,"2"]}]},"status":"success"}` res := &http.Response{Body: io.NopCloser(bytes.NewBufferString(resBody))} - result := qd.parseResponse(context.Background(), &models.Query{}, res) + result := qd.parseResponse(context.Background(), &models.Query{}, res, false) assert.Error(t, result.Error) assert.Equal(t, result.Error.Error(), "no resultType found") }) @@ -43,7 +43,7 @@ func TestQueryData_parseResponse(t *testing.T) { t.Run("resultType is set as empty string before result", func(t *testing.T) { resBody := `{"data":{"resultType":"", "result":[{"metric":{"__name__":"some_name","environment":"some_env","id":"some_id","instance":"some_instance:1234","job":"some_job","name":"another_name","region":"some_region"},"value":[1.1,"2"]}]},"status":"success"}` res := &http.Response{Body: io.NopCloser(bytes.NewBufferString(resBody))} - result := qd.parseResponse(context.Background(), &models.Query{}, res) + result := qd.parseResponse(context.Background(), &models.Query{}, res, false) assert.Error(t, result.Error) assert.Equal(t, result.Error.Error(), "unknown result type: ") }) @@ -51,7 +51,7 @@ func TestQueryData_parseResponse(t *testing.T) { t.Run("resultType is set as empty string after result", func(t *testing.T) { resBody := `{"data":{"result":[{"metric":{"__name__":"some_name","environment":"some_env","id":"some_id","instance":"some_instance:1234","job":"some_job","name":"another_name","region":"some_region"},"value":[1.1,"2"]}],"resultType":""},"status":"success"}` res := &http.Response{Body: io.NopCloser(bytes.NewBufferString(resBody))} - result := qd.parseResponse(context.Background(), &models.Query{}, res) + result := qd.parseResponse(context.Background(), &models.Query{}, res, false) assert.Error(t, result.Error) assert.Equal(t, result.Error.Error(), "unknown result type: ") }) From 2d83feaa5b86fef251a1b209e349fbea0abb9b8c Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Thu, 1 Feb 2024 17:16:03 +0000 Subject: [PATCH 64/84] Add aliases for Grafana Cloud locations so that these pages can be moved (#81731) --- docs/sources/fundamentals/intro-to-prometheus/index.md | 1 + docs/sources/fundamentals/timeseries-dimensions/index.md | 1 + docs/sources/fundamentals/timeseries/index.md | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/sources/fundamentals/intro-to-prometheus/index.md b/docs/sources/fundamentals/intro-to-prometheus/index.md index 3217abc965c..6fa0023c62f 100644 --- a/docs/sources/fundamentals/intro-to-prometheus/index.md +++ b/docs/sources/fundamentals/intro-to-prometheus/index.md @@ -1,6 +1,7 @@ --- aliases: - ../basics/timeseries/ + - /docs/grafana-cloud/introduction/prometheus/ description: Introduction to Prometheus keywords: - grafana diff --git a/docs/sources/fundamentals/timeseries-dimensions/index.md b/docs/sources/fundamentals/timeseries-dimensions/index.md index aeaa2182942..b42fb1482d3 100644 --- a/docs/sources/fundamentals/timeseries-dimensions/index.md +++ b/docs/sources/fundamentals/timeseries-dimensions/index.md @@ -3,6 +3,7 @@ aliases: - ../basics/timeseries-dimensions/ - ../getting-started/timeseries-dimensions/ - ../guides/timeseries-dimensions/ + - /docs/grafana-cloud/introduction/timeseries-dimensions/ description: time series dimensions keywords: - grafana diff --git a/docs/sources/fundamentals/timeseries/index.md b/docs/sources/fundamentals/timeseries/index.md index f95f6e6aa41..c08915ea295 100644 --- a/docs/sources/fundamentals/timeseries/index.md +++ b/docs/sources/fundamentals/timeseries/index.md @@ -1,6 +1,7 @@ --- aliases: - ../basics/timeseries/ + - /docs/grafana-cloud/introduction/timeseries/ description: Introduction to time series keywords: - grafana From 7ab833d28c27b6ff1e173396eca250891c03076b Mon Sep 17 00:00:00 2001 From: "lean.dev" <34773040+leandro-deveikis@users.noreply.github.com> Date: Thu, 1 Feb 2024 14:37:36 -0300 Subject: [PATCH 65/84] Licensing: Redact license when overriden by env variable (#81726) --- pkg/setting/setting.go | 1 + pkg/setting/setting_test.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 58bbc3b6274..8fdda78a12b 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -587,6 +587,7 @@ func RedactedValue(key, value string) string { "ENCRYPTION_KEY", "VAULT_TOKEN", "CLIENT_SECRET", + "ENTERPRISE_LICENSE", } { if match, err := regexp.MatchString(pattern, uppercased); match && err == nil { return RedactedPassword diff --git a/pkg/setting/setting_test.go b/pkg/setting/setting_test.go index 08a20bd0125..a139160329c 100644 --- a/pkg/setting/setting_test.go +++ b/pkg/setting/setting_test.go @@ -830,6 +830,12 @@ func TestRedactedValue(t *testing.T) { value: "/path/to/key", expected: RedactedPassword, }, + { + desc: "license key with non-empty value", + key: "GF_ENTERPRISE_LICENSE_TEXT", + value: "some_license_key_test", + expected: RedactedPassword, + }, { desc: "sensitive key with empty value", key: "private_key_path", From 19ba2114613fcad0302daadf87b152fdb4dd0112 Mon Sep 17 00:00:00 2001 From: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:56:47 +0200 Subject: [PATCH 66/84] Swagger: Update specifications (#81735) --- public/api-enterprise-spec.json | 56 +++++++++++++++++++-------------- public/api-merged.json | 22 ------------- public/openapi3.json | 22 ------------- 3 files changed, 32 insertions(+), 68 deletions(-) diff --git a/public/api-enterprise-spec.json b/public/api-enterprise-spec.json index 3a27c7aa42e..a7d5a6d07b1 100644 --- a/public/api-enterprise-spec.json +++ b/public/api-enterprise-spec.json @@ -6537,24 +6537,6 @@ } } }, - "SSOSettings": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "provider": { - "type": "string" - }, - "settings": { - "type": "object", - "additionalProperties": {} - }, - "source": { - "$ref": "#/definitions/SettingsSource" - } - } - }, "SaveDashboardCommand": { "type": "object", "properties": { @@ -6945,10 +6927,6 @@ } } }, - "SettingsSource": { - "type": "integer", - "format": "int64" - }, "ShareType": { "type": "string" }, @@ -9006,7 +8984,22 @@ "getSSOSettingsResponse": { "description": "", "schema": { - "$ref": "#/definitions/SSOSettings" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "type": "object", + "additionalProperties": {} + }, + "source": { + "type": "string" + } + } } }, "getSharingOptionsResponse": { @@ -9192,7 +9185,22 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/SSOSettings" + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "settings": { + "type": "object", + "additionalProperties": {} + }, + "source": { + "type": "string" + } + } } } }, diff --git a/public/api-merged.json b/public/api-merged.json index ee6218c9f04..2293bfa87c5 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -19280,24 +19280,6 @@ } } }, - "SSOSettings": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "provider": { - "type": "string" - }, - "settings": { - "type": "object", - "additionalProperties": false - }, - "source": { - "$ref": "#/definitions/SettingsSource" - } - } - }, "Sample": { "type": "object", "title": "Sample is a single sample belonging to a metric.", @@ -19716,10 +19698,6 @@ } } }, - "SettingsSource": { - "type": "integer", - "format": "int64" - }, "ShareType": { "type": "string" }, diff --git a/public/openapi3.json b/public/openapi3.json index 3410723e17e..d695f4645dd 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -9796,24 +9796,6 @@ }, "type": "object" }, - "SSOSettings": { - "properties": { - "id": { - "type": "string" - }, - "provider": { - "type": "string" - }, - "settings": { - "additionalProperties": false, - "type": "object" - }, - "source": { - "$ref": "#/components/schemas/SettingsSource" - } - }, - "type": "object" - }, "Sample": { "properties": { "H": { @@ -10231,10 +10213,6 @@ }, "type": "object" }, - "SettingsSource": { - "format": "int64", - "type": "integer" - }, "ShareType": { "type": "string" }, From 780c28be2d1cf8fdc3576338eb3fcfd88833d8dd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:50:22 +0200 Subject: [PATCH 67/84] Update dependency dangerously-set-html-content to v1.1.0 (#80824) * Update dependency dangerously-set-html-content to v1.1.0 * allow rerender and use ' ' for empty state to prevent throwing --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Ashley Harrison --- package.json | 2 +- public/app/plugins/panel/text/TextPanel.tsx | 3 ++- yarn.lock | 14 ++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index f156e1c9730..5d90f6a5f7d 100644 --- a/package.json +++ b/package.json @@ -299,7 +299,7 @@ "d3": "7.8.5", "d3-force": "3.0.0", "d3-scale-chromatic": "3.0.0", - "dangerously-set-html-content": "1.0.9", + "dangerously-set-html-content": "1.1.0", "date-fns": "3.3.1", "debounce-promise": "3.1.2", "diff": "^5.1.0", diff --git a/public/app/plugins/panel/text/TextPanel.tsx b/public/app/plugins/panel/text/TextPanel.tsx index d831532f48a..1d040905cdb 100644 --- a/public/app/plugins/panel/text/TextPanel.tsx +++ b/public/app/plugins/panel/text/TextPanel.tsx @@ -53,6 +53,7 @@ export function TextPanel(props: Props) { return ( Date: Thu, 1 Feb 2024 20:16:07 +0100 Subject: [PATCH 68/84] Chore: Use maputil functions from grafana-azure-sdk-go (#81734) use maputil functions from grafana-azure-sdk-go --- pkg/tsdb/prometheus/client/transport.go | 3 ++- pkg/tsdb/prometheus/querydata/request.go | 2 +- pkg/tsdb/prometheus/resource/resource.go | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/tsdb/prometheus/client/transport.go b/pkg/tsdb/prometheus/client/transport.go index 1776e5ef370..ce7db24e81b 100644 --- a/pkg/tsdb/prometheus/client/transport.go +++ b/pkg/tsdb/prometheus/client/transport.go @@ -5,15 +5,16 @@ import ( "fmt" "strings" + "github.com/grafana/grafana-azure-sdk-go/util/maputil" "github.com/grafana/grafana-plugin-sdk-go/backend" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana-plugin-sdk-go/backend/log" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/prometheus/azureauth" "github.com/grafana/grafana/pkg/tsdb/prometheus/middleware" "github.com/grafana/grafana/pkg/tsdb/prometheus/utils" - "github.com/grafana/grafana/pkg/util/maputil" ) // CreateTransportOptions creates options for the http client. Probably should be shared and should not live in the diff --git a/pkg/tsdb/prometheus/querydata/request.go b/pkg/tsdb/prometheus/querydata/request.go index 1282354207e..41106ba85b3 100644 --- a/pkg/tsdb/prometheus/querydata/request.go +++ b/pkg/tsdb/prometheus/querydata/request.go @@ -7,6 +7,7 @@ import ( "regexp" "time" + "github.com/grafana/grafana-azure-sdk-go/util/maputil" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" "go.opentelemetry.io/otel/attribute" @@ -20,7 +21,6 @@ import ( "github.com/grafana/grafana/pkg/tsdb/prometheus/models" "github.com/grafana/grafana/pkg/tsdb/prometheus/querydata/exemplar" "github.com/grafana/grafana/pkg/tsdb/prometheus/utils" - "github.com/grafana/grafana/pkg/util/maputil" ) const legendFormatAuto = "__auto" diff --git a/pkg/tsdb/prometheus/resource/resource.go b/pkg/tsdb/prometheus/resource/resource.go index 1c5223c3617..aeda2303ea2 100644 --- a/pkg/tsdb/prometheus/resource/resource.go +++ b/pkg/tsdb/prometheus/resource/resource.go @@ -6,11 +6,12 @@ import ( "fmt" "net/http" + "github.com/grafana/grafana-azure-sdk-go/util/maputil" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/log" + "github.com/grafana/grafana/pkg/tsdb/prometheus/client" "github.com/grafana/grafana/pkg/tsdb/prometheus/utils" - "github.com/grafana/grafana/pkg/util/maputil" ) type Resource struct { From 372cf797d128e6339168f262e373abffa508cd9e Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 1 Feb 2024 20:16:21 +0100 Subject: [PATCH 69/84] Chore: Update tests and import orders in prometheus backend (#81732) * use fmt instead of grafana-cli/log * reorder imports * reorder imports --- pkg/tsdb/prometheus/client/client_test.go | 10 +++++----- pkg/tsdb/prometheus/healthcheck.go | 1 + pkg/tsdb/prometheus/querydata/framing_test.go | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/tsdb/prometheus/client/client_test.go b/pkg/tsdb/prometheus/client/client_test.go index 920baf81dfb..1f957571601 100644 --- a/pkg/tsdb/prometheus/client/client_test.go +++ b/pkg/tsdb/prometheus/client/client_test.go @@ -2,6 +2,7 @@ package client import ( "context" + "fmt" "io" "net/http" "testing" @@ -10,7 +11,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/tsdb/prometheus/models" ) @@ -44,7 +44,7 @@ func TestClient(t *testing.T) { defer func() { if res != nil && res.Body != nil { if err := res.Body.Close(); err != nil { - logger.Warn("Error", "err", err) + fmt.Println("Error", "err", err) } } }() @@ -68,7 +68,7 @@ func TestClient(t *testing.T) { defer func() { if res != nil && res.Body != nil { if err := res.Body.Close(); err != nil { - logger.Warn("Error", "err", err) + fmt.Println("Error", "err", err) } } }() @@ -98,7 +98,7 @@ func TestClient(t *testing.T) { defer func() { if res != nil && res.Body != nil { if err := res.Body.Close(); err != nil { - logger.Warn("Error", "err", err) + fmt.Println("Error", "err", err) } } }() @@ -125,7 +125,7 @@ func TestClient(t *testing.T) { defer func() { if res != nil && res.Body != nil { if err := res.Body.Close(); err != nil { - logger.Warn("Error", "err", err) + fmt.Println("Error", "err", err) } } }() diff --git a/pkg/tsdb/prometheus/healthcheck.go b/pkg/tsdb/prometheus/healthcheck.go index 873b8ac06fd..4041d9c08e0 100644 --- a/pkg/tsdb/prometheus/healthcheck.go +++ b/pkg/tsdb/prometheus/healthcheck.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/log" + "github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery" "github.com/grafana/grafana/pkg/tsdb/prometheus/models" ) diff --git a/pkg/tsdb/prometheus/querydata/framing_test.go b/pkg/tsdb/prometheus/querydata/framing_test.go index feadc406619..0bf8ed7284c 100644 --- a/pkg/tsdb/prometheus/querydata/framing_test.go +++ b/pkg/tsdb/prometheus/querydata/framing_test.go @@ -14,9 +14,10 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/experimental" - "github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/tsdb/prometheus/kinds/dataquery" + "github.com/grafana/grafana/pkg/tsdb/prometheus/models" ) From 80ef96e213c868e4a74a73414453b16e016f5fa7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:02:39 +0000 Subject: [PATCH 70/84] Update dependency @glideapps/glide-data-grid to v6.0.2 --- yarn.lock | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/yarn.lock b/yarn.lock index e6c811ad201..b0e6f56209b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3008,8 +3008,8 @@ __metadata: linkType: hard "@glideapps/glide-data-grid@npm:^6.0.0": - version: 6.0.1 - resolution: "@glideapps/glide-data-grid@npm:6.0.1" + version: 6.0.2 + resolution: "@glideapps/glide-data-grid@npm:6.0.2" dependencies: "@linaria/react": "npm:^4.5.3" canvas-hypertxt: "npm:^1.0.3" @@ -3020,7 +3020,7 @@ __metadata: react: ^16.12.0 || 17.x || 18.x react-dom: ^16.12.0 || 17.x || 18.x react-responsive-carousel: ^3.2.7 - checksum: 10/3e99e443442404b91aa9a09791b8bf914d46605b5f9efcc6cb299143fca839d1455d2e392d867f9ed9ddd754885c40be6313c3490efe68b98bec445bdd56130f + checksum: 10/47b9ec7205612f3f1c46558f27d63de7136909b80e4a85e34730941bedefa6e032ed5c4cc56e7a0ebc0b17a9a18a277a0e9c02c068e59f1b4e82b704c9f162b0 languageName: node linkType: hard @@ -8463,14 +8463,7 @@ __metadata: languageName: node linkType: hard -"@types/d3-random@npm:*": - version: 3.0.1 - resolution: "@types/d3-random@npm:3.0.1" - checksum: 10/61f4963bfbfd5cd5284f1f7df10007dca2852bb54c94dd8e6ee9a8094b81d53ed7b24b3889171288f4dea2e251e44eb99f4025cfa5d1dd434bb6265919eb59c9 - languageName: node - linkType: hard - -"@types/d3-random@npm:^3.0.2": +"@types/d3-random@npm:*, @types/d3-random@npm:^3.0.2": version: 3.0.3 resolution: "@types/d3-random@npm:3.0.3" checksum: 10/2c126dda6846f6c7e02c9123a30b4cdf27f3655d19b78456bbb330fbac27acceeeb987318055d3964dba8e6450377ff737db91d81f27c81ca6f4522c9b994ef2 From 7e939401dcd33b04158866251bc3da251c3c00e2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 1 Feb 2024 14:42:59 -0500 Subject: [PATCH 71/84] Alerting: Introduce initial common receiver service (#81211) * Create locking config store that mimics existing provisioning store * Rename existing receivers(_test).go * Introduce shared receiver group service * Fix test * Move query model to models package * ReceiverGroup -> Receiver * Remove locking config store * Move convert methods to compat.go * Cleanup --- .../ngalert/api/api_provisioning_test.go | 3 +- pkg/services/ngalert/models/receivers.go | 10 + pkg/services/ngalert/ngalert.go | 4 +- pkg/services/ngalert/notifier/compat.go | 68 ++++++ pkg/services/ngalert/notifier/receiver_svc.go | 149 +++++++++++++ .../ngalert/notifier/receiver_svc_test.go | 208 ++++++++++++++++++ .../{receivers.go => testreceivers.go} | 0 ...eceivers_test.go => testreceivers_test.go} | 0 pkg/services/ngalert/provisioning/compat.go | 22 ++ .../ngalert/provisioning/contactpoints.go | 86 +++++--- .../provisioning/contactpoints_test.go | 65 ++++-- pkg/services/ngalert/tests/fakes/config.go | 16 +- pkg/services/provisioning/provisioning.go | 4 +- 13 files changed, 577 insertions(+), 58 deletions(-) create mode 100644 pkg/services/ngalert/models/receivers.go create mode 100644 pkg/services/ngalert/notifier/receiver_svc.go create mode 100644 pkg/services/ngalert/notifier/receiver_svc_test.go rename pkg/services/ngalert/notifier/{receivers.go => testreceivers.go} (100%) rename pkg/services/ngalert/notifier/{receivers_test.go => testreceivers_test.go} (100%) diff --git a/pkg/services/ngalert/api/api_provisioning_test.go b/pkg/services/ngalert/api/api_provisioning_test.go index ded0b5196d6..376d838de7f 100644 --- a/pkg/services/ngalert/api/api_provisioning_test.go +++ b/pkg/services/ngalert/api/api_provisioning_test.go @@ -1614,10 +1614,11 @@ func createProvisioningSrvSut(t *testing.T) ProvisioningSrv { func createProvisioningSrvSutFromEnv(t *testing.T, env *testEnvironment) ProvisioningSrv { t.Helper() + receiverSvc := notifier.NewReceiverService(env.ac, env.configs, env.prov, env.secrets, env.xact, env.log) return ProvisioningSrv{ log: env.log, policies: newFakeNotificationPolicyService(), - contactPointService: provisioning.NewContactPointService(env.configs, env.secrets, env.prov, env.xact, env.log, env.ac), + contactPointService: provisioning.NewContactPointService(env.configs, env.secrets, env.prov, env.xact, receiverSvc, env.log), templates: provisioning.NewTemplateService(env.configs, env.prov, env.xact, env.log), muteTimings: provisioning.NewMuteTimingService(env.configs, env.prov, env.xact, env.log), alertRules: provisioning.NewAlertRuleService(env.store, env.prov, env.dashboardService, env.quotas, env.xact, 60, 10, env.log), diff --git a/pkg/services/ngalert/models/receivers.go b/pkg/services/ngalert/models/receivers.go new file mode 100644 index 00000000000..048955428bc --- /dev/null +++ b/pkg/services/ngalert/models/receivers.go @@ -0,0 +1,10 @@ +package models + +// GetReceiversQuery represents a query for receiver groups. +type GetReceiversQuery struct { + OrgID int64 + Names []string + Limit int + Offset int + Decrypt bool +} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index e4e610400ab..22268ab993f 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -313,9 +313,11 @@ func (ng *AlertNG) init() error { ng.stateManager = stateManager ng.schedule = scheduler + receiverService := notifier.NewReceiverService(ng.accesscontrol, ng.store, ng.store, ng.SecretsService, ng.store, ng.Log) + // Provisioning policyService := provisioning.NewNotificationPolicyService(ng.store, ng.store, ng.store, ng.Cfg.UnifiedAlerting, ng.Log) - contactPointService := provisioning.NewContactPointService(ng.store, ng.SecretsService, ng.store, ng.store, ng.Log, ng.accesscontrol) + contactPointService := provisioning.NewContactPointService(ng.store, ng.SecretsService, ng.store, ng.store, receiverService, ng.Log) templateService := provisioning.NewTemplateService(ng.store, ng.store, ng.store, ng.Log) muteTimingService := provisioning.NewMuteTimingService(ng.store, ng.store, ng.store, ng.Log) alertRuleService := provisioning.NewAlertRuleService(ng.store, ng.store, ng.dashboardService, ng.QuotaService, ng.store, diff --git a/pkg/services/ngalert/notifier/compat.go b/pkg/services/ngalert/notifier/compat.go index 41bcb1cab87..1731d384416 100644 --- a/pkg/services/ngalert/notifier/compat.go +++ b/pkg/services/ngalert/notifier/compat.go @@ -4,8 +4,11 @@ import ( "encoding/json" alertingNotify "github.com/grafana/alerting/notify" + "github.com/prometheus/alertmanager/config" + "github.com/grafana/grafana/pkg/components/simplejson" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/models" ) func PostableGrafanaReceiverToGrafanaIntegrationConfig(p *apimodels.PostableGrafanaReceiver) *alertingNotify.GrafanaIntegrationConfig { @@ -40,3 +43,68 @@ func PostableApiAlertingConfigToApiReceivers(c apimodels.PostableApiAlertingConf } return apiReceivers } + +type DecryptFn = func(value string) string + +func PostableToGettableGrafanaReceiver(r *apimodels.PostableGrafanaReceiver, provenance *models.Provenance, decryptFn DecryptFn) (apimodels.GettableGrafanaReceiver, error) { + out := apimodels.GettableGrafanaReceiver{ + UID: r.UID, + Name: r.Name, + Type: r.Type, + DisableResolveMessage: r.DisableResolveMessage, + SecureFields: make(map[string]bool, len(r.SecureSettings)), + } + + if provenance != nil { + out.Provenance = apimodels.Provenance(*provenance) + } + + settings, err := simplejson.NewJson([]byte(r.Settings)) + if err != nil { + return apimodels.GettableGrafanaReceiver{}, err + } + + for k, v := range r.SecureSettings { + decryptedValue := decryptFn(v) + if err != nil { + return apimodels.GettableGrafanaReceiver{}, err + } + if decryptedValue == "" { + continue + } else { + settings.Set(k, decryptedValue) + } + out.SecureFields[k] = true + } + + jsonBytes, err := settings.MarshalJSON() + if err != nil { + return apimodels.GettableGrafanaReceiver{}, err + } + out.Settings = jsonBytes + + return out, nil +} + +func PostableToGettableApiReceiver(r *apimodels.PostableApiReceiver, provenances map[string]models.Provenance, decryptFn DecryptFn) (apimodels.GettableApiReceiver, error) { + out := apimodels.GettableApiReceiver{ + Receiver: config.Receiver{ + Name: r.Receiver.Name, + }, + } + + for _, gr := range r.GrafanaManagedReceivers { + var prov *models.Provenance + if p, ok := provenances[gr.UID]; ok { + prov = &p + } + + gettable, err := PostableToGettableGrafanaReceiver(gr, prov, decryptFn) + if err != nil { + return apimodels.GettableApiReceiver{}, err + } + out.GrafanaManagedReceivers = append(out.GrafanaManagedReceivers, &gettable) + } + + return out, nil +} diff --git a/pkg/services/ngalert/notifier/receiver_svc.go b/pkg/services/ngalert/notifier/receiver_svc.go new file mode 100644 index 00000000000..cefa1ffd64a --- /dev/null +++ b/pkg/services/ngalert/notifier/receiver_svc.go @@ -0,0 +1,149 @@ +package notifier + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "slices" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/auth/identity" + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/secrets" +) + +var ( + // ErrPermissionDenied is returned when the user does not have permission to perform the requested action. + ErrPermissionDenied = errors.New("permission denied") // TODO: convert to errutil +) + +// ReceiverService is the service for managing alertmanager receivers. +type ReceiverService struct { + ac accesscontrol.AccessControl + provisioningStore provisoningStore + cfgStore configStore + encryptionService secrets.Service + xact transactionManager + log log.Logger +} + +type configStore interface { + GetLatestAlertmanagerConfiguration(ctx context.Context, orgID int64) (*models.AlertConfiguration, error) + UpdateAlertmanagerConfiguration(ctx context.Context, cmd *models.SaveAlertmanagerConfigurationCmd) error +} + +type provisoningStore interface { + GetProvenances(ctx context.Context, org int64, resourceType string) (map[string]models.Provenance, error) +} + +type transactionManager interface { + InTransaction(ctx context.Context, work func(ctx context.Context) error) error +} + +func NewReceiverService( + ac accesscontrol.AccessControl, + cfgStore configStore, + provisioningStore provisoningStore, + encryptionService secrets.Service, + xact transactionManager, + log log.Logger, +) *ReceiverService { + return &ReceiverService{ + ac: ac, + provisioningStore: provisioningStore, + cfgStore: cfgStore, + encryptionService: encryptionService, + xact: xact, + log: log, + } +} + +func (rs *ReceiverService) canDecrypt(ctx context.Context, user identity.Requester, name string) (bool, error) { + receiverAccess := false // TODO: stub, check for read secrets access + eval := accesscontrol.EvalPermission(accesscontrol.ActionAlertingProvisioningReadSecrets) + provisioningAccess, err := rs.ac.Evaluate(ctx, user, eval) + if err != nil { + return false, err + } + return receiverAccess || provisioningAccess, nil +} + +// GetReceivers returns a list of receivers a user has access to. +// Receivers can be filtered by name, and secure settings are decrypted if requested and the user has access to do so. +func (rs *ReceiverService) GetReceivers(ctx context.Context, q models.GetReceiversQuery, user identity.Requester) ([]definitions.GettableApiReceiver, error) { + baseCfg, err := rs.cfgStore.GetLatestAlertmanagerConfiguration(ctx, q.OrgID) + if err != nil { + return nil, err + } + + cfg := definitions.PostableUserConfig{} + err = json.Unmarshal([]byte(baseCfg.AlertmanagerConfiguration), &cfg) + if err != nil { + return nil, err + } + + provenances, err := rs.provisioningStore.GetProvenances(ctx, q.OrgID, "contactPoint") + if err != nil { + return nil, err + } + + // TODO: check for list access + + var output []definitions.GettableApiReceiver + for i := q.Offset; i < len(cfg.AlertmanagerConfig.Receivers); i++ { + r := cfg.AlertmanagerConfig.Receivers[i] + if len(q.Names) > 0 && !slices.Contains(q.Names, r.Name) { + continue + } + + // TODO: check for scoped read access and continue if not allowed + + decryptAccess, err := rs.canDecrypt(ctx, user, r.Name) + if err != nil { + return nil, err + } + if q.Decrypt && !decryptAccess { + return nil, ErrPermissionDenied + } + + decryptFn := rs.decryptOrRedact(ctx, decryptAccess && q.Decrypt, r.Name, "") + + res, err := PostableToGettableApiReceiver(r, provenances, decryptFn) + if err != nil { + return nil, err + } + + // TODO: redact settings if the user only has list access + + output = append(output, res) + // stop if we have reached the limit or we have found all the requested receivers + if (len(output) == q.Limit && q.Limit > 0) || (len(output) == len(q.Names)) { + break + } + } + + return output, nil +} + +func (rs *ReceiverService) decryptOrRedact(ctx context.Context, decrypt bool, name, fallback string) func(value string) string { + return func(value string) string { + if !decrypt { + return definitions.RedactedValue + } + + decoded, err := base64.StdEncoding.DecodeString(value) + if err != nil { + rs.log.Warn("failed to decode secure setting", "name", name, "error", err) + return fallback + } + decrypted, err := rs.encryptionService.Decrypt(ctx, decoded) + if err != nil { + rs.log.Warn("failed to decrypt secure setting", "name", name, "error", err) + return fallback + } + return string(decrypted) + } +} diff --git a/pkg/services/ngalert/notifier/receiver_svc_test.go b/pkg/services/ngalert/notifier/receiver_svc_test.go new file mode 100644 index 00000000000..86426192767 --- /dev/null +++ b/pkg/services/ngalert/notifier/receiver_svc_test.go @@ -0,0 +1,208 @@ +package notifier + +import ( + "context" + "encoding/json" + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/secrets/database" + "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/require" +) + +func TestReceiverService_GetReceivers(t *testing.T) { + sqlStore := db.InitTestDB(t) + secretsService := manager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) + + t.Run("service gets receiver groups from AM config", func(t *testing.T) { + sut := createReceiverServiceSut(t, secretsService) + + Receivers, err := sut.GetReceivers(context.Background(), rQuery(1), nil) + require.NoError(t, err) + require.Len(t, Receivers, 2) + require.Equal(t, "grafana-default-email", Receivers[0].Name) + require.Equal(t, "slack receiver", Receivers[1].Name) + }) + + t.Run("service filters receiver groups by name", func(t *testing.T) { + sut := createReceiverServiceSut(t, secretsService) + + Receivers, err := sut.GetReceivers(context.Background(), rQuery(1, "slack receiver"), nil) + require.NoError(t, err) + require.Len(t, Receivers, 1) + require.Equal(t, "slack receiver", Receivers[0].Name) + }) +} + +func TestReceiverService_DecryptRedact(t *testing.T) { + sqlStore := db.InitTestDB(t) + secretsService := manager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) + ac := acimpl.ProvideAccessControl(setting.NewCfg()) + + t.Run("service redacts receiver groups by default", func(t *testing.T) { + sut := createReceiverServiceSut(t, secretsService) + Receivers, err := sut.GetReceivers(context.Background(), rQuery(1, "slack receiver"), nil) + require.NoError(t, err) + + require.Len(t, Receivers, 1) + + rGroup := Receivers[0] + require.Equal(t, "slack receiver", rGroup.Name) + require.Len(t, rGroup.GrafanaManagedReceivers, 1) + + grafanaReceiver := rGroup.GrafanaManagedReceivers[0] + require.Equal(t, "UID2", grafanaReceiver.UID) + + testedSettings, err := simplejson.NewJson([]byte(grafanaReceiver.Settings)) + require.NoError(t, err) + require.Equal(t, definitions.RedactedValue, testedSettings.Get("url").MustString()) + }) + + t.Run("service returns error when trying to decrypt with nil user", func(t *testing.T) { + sut := createReceiverServiceSut(t, secretsService) + sut.ac = ac + + q := rQuery(1) + q.Decrypt = true + _, err := sut.GetReceivers(context.Background(), q, nil) + require.ErrorIs(t, err, ErrPermissionDenied) + }) + + t.Run("service returns error when trying to decrypt without permission", func(t *testing.T) { + sut := createReceiverServiceSut(t, secretsService) + sut.ac = ac + + q := rQuery(1) + q.Decrypt = true + _, err := sut.GetReceivers(context.Background(), q, &user.SignedInUser{}) + require.ErrorIs(t, err, ErrPermissionDenied) + }) + + t.Run("service decrypts receiver groups with permission", func(t *testing.T) { + sut := createReceiverServiceSut(t, secretsService) + sut.ac = ac + + q := rQuery(1, "slack receiver") + q.Decrypt = true + Receivers, err := sut.GetReceivers(context.Background(), q, &user.SignedInUser{ + OrgID: 1, + Permissions: map[int64]map[string][]string{ + 1: {accesscontrol.ActionAlertingProvisioningReadSecrets: nil}, + }, + }) + require.NoError(t, err) + + require.Len(t, Receivers, 1) + + rGroup := Receivers[0] + require.Equal(t, "slack receiver", rGroup.Name) + require.Len(t, rGroup.GrafanaManagedReceivers, 1) + + grafanaReceiver := rGroup.GrafanaManagedReceivers[0] + require.Equal(t, "UID2", grafanaReceiver.UID) + + settings, err := simplejson.NewJson([]byte(grafanaReceiver.Settings)) + require.NoError(t, err) + require.Equal(t, "secure url", settings.Get("url").MustString()) + }) +} + +func createReceiverServiceSut(t *testing.T, encryptSvc secrets.Service) *ReceiverService { + cfg := createEncryptedConfig(t, encryptSvc) + store := fakes.NewFakeAlertmanagerConfigStore(cfg) + xact := newNopTransactionManager() + provisioningStore := fakes.NewFakeProvisioningStore() + + return &ReceiverService{ + actest.FakeAccessControl{}, + provisioningStore, + store, + encryptSvc, + xact, + log.NewNopLogger(), + } +} + +func createEncryptedConfig(t *testing.T, secretService secrets.Service) string { + c := &definitions.PostableUserConfig{} + err := json.Unmarshal([]byte(defaultAlertmanagerConfigJSON), c) + require.NoError(t, err) + err = EncryptReceiverConfigs(c.AlertmanagerConfig.Receivers, func(ctx context.Context, payload []byte) ([]byte, error) { + return secretService.Encrypt(ctx, payload, secrets.WithoutScope()) + }) + require.NoError(t, err) + bytes, err := json.Marshal(c) + require.NoError(t, err) + return string(bytes) +} + +func rQuery(orgID int64, names ...string) models.GetReceiversQuery { + return models.GetReceiversQuery{ + OrgID: orgID, + Names: names, + } +} + +const defaultAlertmanagerConfigJSON = ` +{ + "template_files": null, + "alertmanager_config": { + "route": { + "receiver": "grafana-default-email", + "group_by": [ + "..." + ], + "routes": [{ + "receiver": "grafana-default-email", + "object_matchers": [["a", "=", "b"]] + }] + }, + "templates": null, + "receivers": [{ + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [{ + "uid": "UID1", + "name": "grafana-default-email", + "type": "email", + "disableResolveMessage": false, + "settings": { + "addresses": "\u003cexample@email.com\u003e" + }, + "secureFields": {} + }] + }, { + "name": "slack receiver", + "grafana_managed_receiver_configs": [{ + "uid": "UID2", + "name": "slack receiver", + "type": "slack", + "disableResolveMessage": false, + "settings": {}, + "secureSettings": {"url":"secure url"} + }] + }] + } +} +` + +type NopTransactionManager struct{} + +func newNopTransactionManager() *NopTransactionManager { + return &NopTransactionManager{} +} + +func (n *NopTransactionManager) InTransaction(ctx context.Context, work func(ctx context.Context) error) error { + return work(context.WithValue(ctx, NopTransactionManager{}, struct{}{})) +} diff --git a/pkg/services/ngalert/notifier/receivers.go b/pkg/services/ngalert/notifier/testreceivers.go similarity index 100% rename from pkg/services/ngalert/notifier/receivers.go rename to pkg/services/ngalert/notifier/testreceivers.go diff --git a/pkg/services/ngalert/notifier/receivers_test.go b/pkg/services/ngalert/notifier/testreceivers_test.go similarity index 100% rename from pkg/services/ngalert/notifier/receivers_test.go rename to pkg/services/ngalert/notifier/testreceivers_test.go diff --git a/pkg/services/ngalert/provisioning/compat.go b/pkg/services/ngalert/provisioning/compat.go index 621ce88b4f5..0502d921d8d 100644 --- a/pkg/services/ngalert/provisioning/compat.go +++ b/pkg/services/ngalert/provisioning/compat.go @@ -45,3 +45,25 @@ func PostableGrafanaReceiverToEmbeddedContactPoint(contactPoint *definitions.Pos } return embeddedContactPoint, nil } + +func GettableGrafanaReceiverToEmbeddedContactPoint(r *definitions.GettableGrafanaReceiver) (definitions.EmbeddedContactPoint, error) { + settingJson, err := simplejson.NewJson(r.Settings) + if err != nil { + return definitions.EmbeddedContactPoint{}, err + } + + for k := range r.SecureFields { + if settingJson.Get(k).MustString() == "" { + settingJson.Set(k, definitions.RedactedValue) + } + } + + return definitions.EmbeddedContactPoint{ + UID: r.UID, + Name: r.Name, + Type: r.Type, + DisableResolveMessage: r.DisableResolveMessage, + Settings: settingJson, + Provenance: string(r.Provenance), + }, nil +} diff --git a/pkg/services/ngalert/provisioning/contactpoints.go b/pkg/services/ngalert/provisioning/contactpoints.go index c5dd067682a..92142fb363c 100644 --- a/pkg/services/ngalert/provisioning/contactpoints.go +++ b/pkg/services/ngalert/provisioning/contactpoints.go @@ -12,11 +12,12 @@ import ( "github.com/prometheus/alertmanager/config" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/auth/identity" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels_config" + "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/util" ) @@ -26,21 +27,25 @@ type ContactPointService struct { encryptionService secrets.Service provenanceStore ProvisioningStore xact TransactionManager + receiverService receiverService log log.Logger - ac accesscontrol.AccessControl +} + +type receiverService interface { + GetReceivers(ctx context.Context, query models.GetReceiversQuery, user identity.Requester) ([]apimodels.GettableApiReceiver, error) } func NewContactPointService(store AMConfigStore, encryptionService secrets.Service, - provenanceStore ProvisioningStore, xact TransactionManager, log log.Logger, ac accesscontrol.AccessControl) *ContactPointService { + provenanceStore ProvisioningStore, xact TransactionManager, receiverService receiverService, log log.Logger) *ContactPointService { return &ContactPointService{ configStore: &alertmanagerConfigStoreImpl{ store: store, }, + receiverService: receiverService, encryptionService: encryptionService, provenanceStore: provenanceStore, xact: xact, log: log, - ac: ac, } } @@ -52,48 +57,38 @@ type ContactPointQuery struct { Decrypt bool } -func (ecp *ContactPointService) canDecryptSecrets(ctx context.Context, u identity.Requester) bool { - if u == nil { - return false - } - permitted, err := ecp.ac.Evaluate(ctx, u, accesscontrol.EvalPermission(accesscontrol.ActionAlertingProvisioningReadSecrets)) - if err != nil { - ecp.log.Error("Failed to evaluate user permissions", "error", err) - permitted = false - } - return permitted -} - // GetContactPoints returns contact points. If q.Decrypt is true and the user is an OrgAdmin, decrypted secure settings are included instead of redacted ones. func (ecp *ContactPointService) GetContactPoints(ctx context.Context, q ContactPointQuery, u identity.Requester) ([]apimodels.EmbeddedContactPoint, error) { - if q.Decrypt && !ecp.canDecryptSecrets(ctx, u) { - return nil, fmt.Errorf("%w: user requires Admin role or alert.provisioning.secrets:read permission to view decrypted secure settings", ErrPermissionDenied) + receiverQuery := models.GetReceiversQuery{ + OrgID: q.OrgID, + Decrypt: q.Decrypt, } - revision, err := ecp.configStore.Get(ctx, q.OrgID) - if err != nil { - return nil, err + if q.Name != "" { + receiverQuery.Names = []string{q.Name} } - provenances, err := ecp.provenanceStore.GetProvenances(ctx, q.OrgID, "contactPoint") - if err != nil { - return nil, err - } - var contactPoints []apimodels.EmbeddedContactPoint - for _, contactPoint := range revision.cfg.GetGrafanaReceiverMap() { - if q.Name != "" && contactPoint.Name != q.Name { - continue + res, err := ecp.receiverService.GetReceivers(ctx, receiverQuery, u) + if err != nil { + return nil, convertRecSvcErr(err) + } + grafanaReceivers := []*apimodels.GettableGrafanaReceiver{} + if q.Name != "" { + grafanaReceivers = res[0].GettableGrafanaReceivers.GrafanaManagedReceivers // we only expect one receiver group + } else { + for _, r := range res { + grafanaReceivers = append(grafanaReceivers, r.GettableGrafanaReceivers.GrafanaManagedReceivers...) } + } - embeddedContactPoint, err := PostableGrafanaReceiverToEmbeddedContactPoint( - contactPoint, - provenances[contactPoint.UID], - ecp.decryptValueOrRedacted(q.Decrypt, contactPoint.UID), - ) + var contactPoints []apimodels.EmbeddedContactPoint + for _, gr := range grafanaReceivers { + contactPoint, err := GettableGrafanaReceiverToEmbeddedContactPoint(gr) if err != nil { return nil, err } - contactPoints = append(contactPoints, embeddedContactPoint) + contactPoints = append(contactPoints, contactPoint) } + sort.SliceStable(contactPoints, func(i, j int) bool { switch strings.Compare(contactPoints[i].Name, contactPoints[j].Name) { case -1: @@ -103,6 +98,7 @@ func (ecp *ContactPointService) GetContactPoints(ctx context.Context, q ContactP } return contactPoints[i].UID < contactPoints[j].UID }) + return contactPoints, nil } @@ -507,3 +503,23 @@ func RemoveSecretsForContactPoint(e *apimodels.EmbeddedContactPoint) (map[string } return s, nil } + +// handleWrappedError unwraps an error and wraps it with a new expected error type. If the error is not wrapped, it returns just the expected error. +func handleWrappedError(err error, expected error) error { + err = errors.Unwrap(err) + if err == nil { + return expected + } + return fmt.Errorf("%w: %s", expected, err.Error()) +} + +// convertRecSvcErr converts errors from notifier.ReceiverService to errors expected from ContactPointService. +func convertRecSvcErr(err error) error { + if errors.Is(err, notifier.ErrPermissionDenied) { + return handleWrappedError(err, ErrPermissionDenied) + } + if errors.Is(err, store.ErrNoAlertmanagerConfiguration) { + return ErrNoAlertmanagerConfiguration.Errorf("") + } + return err +} diff --git a/pkg/services/ngalert/provisioning/contactpoints_test.go b/pkg/services/ngalert/provisioning/contactpoints_test.go index 38f9bfa983b..6cd33217446 100644 --- a/pkg/services/ngalert/provisioning/contactpoints_test.go +++ b/pkg/services/ngalert/provisioning/contactpoints_test.go @@ -248,9 +248,19 @@ func TestContactPointService(t *testing.T) { } func TestContactPointServiceDecryptRedact(t *testing.T) { - sqlStore := db.InitTestDB(t) - secretsService := manager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) - ac := acimpl.ProvideAccessControl(setting.NewCfg()) + secretsService := manager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t))) + receiverServiceWithAC := func(ecp *ContactPointService) *notifier.ReceiverService { + return notifier.NewReceiverService( + acimpl.ProvideAccessControl(setting.NewCfg()), + // Get won't use the sut's config store, so we can use a different one here. + fakes.NewFakeAlertmanagerConfigStore(createEncryptedConfig(t, secretsService)), + ecp.provenanceStore, + ecp.encryptionService, + ecp.xact, + log.NewNopLogger(), + ) + } + t.Run("GetContactPoints gets redacted contact points by default", func(t *testing.T) { sut := createContactPointServiceSut(t, secretsService) @@ -261,9 +271,10 @@ func TestContactPointServiceDecryptRedact(t *testing.T) { require.Equal(t, "slack receiver", cps[1].Name) require.Equal(t, definitions.RedactedValue, cps[1].Settings.Get("url").MustString()) }) + t.Run("GetContactPoints errors when Decrypt = true and user does not have permissions", func(t *testing.T) { sut := createContactPointServiceSut(t, secretsService) - sut.ac = ac + sut.receiverService = receiverServiceWithAC(sut) q := cpsQuery(1) q.Decrypt = true @@ -272,7 +283,7 @@ func TestContactPointServiceDecryptRedact(t *testing.T) { }) t.Run("GetContactPoints errors when Decrypt = true and user is nil", func(t *testing.T) { sut := createContactPointServiceSut(t, secretsService) - sut.ac = ac + sut.receiverService = receiverServiceWithAC(sut) q := cpsQuery(1) q.Decrypt = true @@ -282,7 +293,7 @@ func TestContactPointServiceDecryptRedact(t *testing.T) { t.Run("GetContactPoints gets decrypted contact points when Decrypt = true and user has permissions", func(t *testing.T) { sut := createContactPointServiceSut(t, secretsService) - sut.ac = ac + sut.receiverService = receiverServiceWithAC(sut) expectedName := "slack receiver" q := cpsQueryWithName(1, expectedName) @@ -333,24 +344,27 @@ func TestContactPointInUse(t *testing.T) { func createContactPointServiceSut(t *testing.T, secretService secrets.Service) *ContactPointService { // Encrypt secure settings. - c := &definitions.PostableUserConfig{} - err := json.Unmarshal([]byte(defaultAlertmanagerConfigJSON), c) - require.NoError(t, err) - err = notifier.EncryptReceiverConfigs(c.AlertmanagerConfig.Receivers, func(ctx context.Context, payload []byte) ([]byte, error) { - return secretService.Encrypt(ctx, payload, secrets.WithoutScope()) - }) - require.NoError(t, err) + cfg := createEncryptedConfig(t, secretService) + store := fakes.NewFakeAlertmanagerConfigStore(cfg) + xact := newNopTransactionManager() + provisioningStore := fakes.NewFakeProvisioningStore() - raw, err := json.Marshal(c) - require.NoError(t, err) + receiverService := notifier.NewReceiverService( + actest.FakeAccessControl{}, + store, + provisioningStore, + secretService, + xact, + log.NewNopLogger(), + ) return &ContactPointService{ - configStore: &alertmanagerConfigStoreImpl{store: fakes.NewFakeAlertmanagerConfigStore(string(raw))}, - provenanceStore: fakes.NewFakeProvisioningStore(), - xact: newNopTransactionManager(), + configStore: &alertmanagerConfigStoreImpl{store: store}, + provenanceStore: provisioningStore, + receiverService: receiverService, + xact: xact, encryptionService: secretService, log: log.NewNopLogger(), - ac: actest.FakeAccessControl{}, } } @@ -376,6 +390,19 @@ func cpsQueryWithName(orgID int64, name string) ContactPointQuery { } } +func createEncryptedConfig(t *testing.T, secretService secrets.Service) string { + c := &definitions.PostableUserConfig{} + err := json.Unmarshal([]byte(defaultAlertmanagerConfigJSON), c) + require.NoError(t, err) + err = notifier.EncryptReceiverConfigs(c.AlertmanagerConfig.Receivers, func(ctx context.Context, payload []byte) ([]byte, error) { + return secretService.Encrypt(ctx, payload, secrets.WithoutScope()) + }) + require.NoError(t, err) + bytes, err := json.Marshal(c) + require.NoError(t, err) + return string(bytes) +} + func TestStitchReceivers(t *testing.T) { type testCase struct { name string diff --git a/pkg/services/ngalert/tests/fakes/config.go b/pkg/services/ngalert/tests/fakes/config.go index 5fe396556ba..c77450f6ca5 100644 --- a/pkg/services/ngalert/tests/fakes/config.go +++ b/pkg/services/ngalert/tests/fakes/config.go @@ -9,7 +9,12 @@ import ( ) type FakeAlertmanagerConfigStore struct { - Config models.AlertConfiguration + Config models.AlertConfiguration + // GetFn is an optional function that can be set to mock the GetLatestAlertmanagerConfiguration method + GetFn func(ctx context.Context, orgID int64) (*models.AlertConfiguration, error) + // UpdateFn is an optional function that can be set to mock the UpdateAlertmanagerConfiguration method + UpdateFn func(ctx context.Context, cmd *models.SaveAlertmanagerConfigurationCmd) error + // LastSaveCommand is the last command that was passed to UpdateAlertmanagerConfiguration LastSaveCommand *models.SaveAlertmanagerConfigurationCmd } @@ -26,13 +31,22 @@ func NewFakeAlertmanagerConfigStore(config string) *FakeAlertmanagerConfigStore } func (f *FakeAlertmanagerConfigStore) GetLatestAlertmanagerConfiguration(ctx context.Context, orgID int64) (*models.AlertConfiguration, error) { + if f.GetFn != nil { + return f.GetFn(ctx, orgID) + } + result := &f.Config result.OrgID = orgID result.ConfigurationHash = fmt.Sprintf("%x", md5.Sum([]byte(f.Config.AlertmanagerConfiguration))) + result.ConfigurationVersion = f.Config.ConfigurationVersion return result, nil } func (f *FakeAlertmanagerConfigStore) UpdateAlertmanagerConfiguration(ctx context.Context, cmd *models.SaveAlertmanagerConfigurationCmd) error { + if f.UpdateFn != nil { + return f.UpdateFn(ctx, cmd) + } + f.Config = models.AlertConfiguration{ AlertmanagerConfiguration: cmd.AlertmanagerConfiguration, ConfigurationVersion: cmd.ConfigurationVersion, diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index 47e5fad7916..2f35b9d60e9 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -16,6 +16,7 @@ import ( datasourceservice "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/encryption" "github.com/grafana/grafana/pkg/services/folder" + alertingNotifier "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/services/notifications" @@ -280,8 +281,9 @@ func (ps *ProvisioningServiceImpl) ProvisionAlerting(ctx context.Context) error int64(ps.Cfg.UnifiedAlerting.DefaultRuleEvaluationInterval.Seconds()), int64(ps.Cfg.UnifiedAlerting.BaseInterval.Seconds()), ps.log) + receiverSvc := alertingNotifier.NewReceiverService(ps.ac, &st, st, ps.secretService, ps.SQLStore, ps.log) contactPointService := provisioning.NewContactPointService(&st, ps.secretService, - st, ps.SQLStore, ps.log, ps.ac) + st, ps.SQLStore, receiverSvc, ps.log) notificationPolicyService := provisioning.NewNotificationPolicyService(&st, st, ps.SQLStore, ps.Cfg.UnifiedAlerting, ps.log) mutetimingsService := provisioning.NewMuteTimingService(&st, st, &st, ps.log) From d1073deefd9d0bbbb5e72a2a21730736c206efa0 Mon Sep 17 00:00:00 2001 From: Yuri Tseretyan Date: Thu, 1 Feb 2024 15:17:13 -0500 Subject: [PATCH 72/84] Alerting: Time intervals API (read only endpoints) (#81672) * declare new API and models GettableTimeIntervals, PostableTimeIntervals * add new actions alert.notifications.time-intervals:read and alert.notifications.time-intervals:write. * update existing alerting roles with the read action. Add to all alerting roles. * add integration tests --- pkg/services/accesscontrol/models.go | 4 + pkg/services/ngalert/accesscontrol.go | 6 + pkg/services/ngalert/api/api.go | 2 + pkg/services/ngalert/api/authorization.go | 3 + .../ngalert/api/authorization_test.go | 2 +- .../api/generated_base_api_notifications.go | 62 ++++++ pkg/services/ngalert/api/notifications.go | 34 +++ pkg/services/ngalert/api/tooling/api.json | 166 +++++++++++++- .../api/tooling/definitions/time_intervals.go | 64 ++++++ pkg/services/ngalert/api/tooling/post.json | 170 ++++++++++++++- pkg/services/ngalert/api/tooling/spec.json | 172 ++++++++++++++- .../api_notifications_time_interval_test.go | 204 ++++++++++++++++++ pkg/tests/api/alerting/testing.go | 18 ++ public/api-merged.json | 154 ++++++++++++- public/openapi3.json | 176 ++++++++++++++- 15 files changed, 1216 insertions(+), 21 deletions(-) create mode 100644 pkg/services/ngalert/api/generated_base_api_notifications.go create mode 100644 pkg/services/ngalert/api/notifications.go create mode 100644 pkg/services/ngalert/api/tooling/definitions/time_intervals.go create mode 100644 pkg/tests/api/alerting/api_notifications_time_interval_test.go diff --git a/pkg/services/accesscontrol/models.go b/pkg/services/accesscontrol/models.go index 68af6d37ddb..b3ee788c439 100644 --- a/pkg/services/accesscontrol/models.go +++ b/pkg/services/accesscontrol/models.go @@ -445,6 +445,10 @@ const ( ActionAlertingNotificationsRead = "alert.notifications:read" ActionAlertingNotificationsWrite = "alert.notifications:write" + // Alerting notifications time interval actions + ActionAlertingNotificationsTimeIntervalsRead = "alert.notifications.time-intervals:read" + ActionAlertingNotificationsTimeIntervalsWrite = "alert.notifications.time-intervals:write" + // External alerting rule actions. We can only narrow it down to writes or reads, as we don't control the atomicity in the external system. ActionAlertingRuleExternalWrite = "alert.rules.external:write" ActionAlertingRuleExternalRead = "alert.rules.external:read" diff --git a/pkg/services/ngalert/accesscontrol.go b/pkg/services/ngalert/accesscontrol.go index 3e9ce4a3f73..a8ab39c0111 100644 --- a/pkg/services/ngalert/accesscontrol.go +++ b/pkg/services/ngalert/accesscontrol.go @@ -25,6 +25,9 @@ var ( Action: accesscontrol.ActionAlertingRuleExternalRead, Scope: datasources.ScopeAll, }, + { + Action: accesscontrol.ActionAlertingNotificationsTimeIntervalsRead, // This is needed for simplified notification policies + }, }, }, } @@ -109,6 +112,9 @@ var ( Action: accesscontrol.ActionAlertingNotificationsExternalRead, Scope: datasources.ScopeAll, }, + { + Action: accesscontrol.ActionAlertingNotificationsTimeIntervalsRead, + }, }, }, } diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index dcb8aec5e48..58bf3a0635d 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -152,6 +152,8 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) { hist: api.Historian, }), m) + api.RegisterNotificationsApiEndpoints(NewNotificationsApi(api.MuteTimings), m) + // Inject upgrade endpoints if legacy alerting is enabled and the feature flag is enabled. if !api.Cfg.UnifiedAlerting.IsEnabled() && api.FeatureManager.IsEnabledGlobally(featuremgmt.FlagAlertingPreviewUpgrade) { api.RegisterUpgradeApiEndpoints(NewUpgradeApi(NewUpgradeSrc( diff --git a/pkg/services/ngalert/api/authorization.go b/pkg/services/ngalert/api/authorization.go index d27ce6ca8f9..95fd74e1b4d 100644 --- a/pkg/services/ngalert/api/authorization.go +++ b/pkg/services/ngalert/api/authorization.go @@ -238,6 +238,9 @@ func (api *API) authorize(method, path string) web.Handler { http.MethodDelete + "/api/v1/provisioning/alert-rules/{UID}", http.MethodPut + "/api/v1/provisioning/folder/{FolderUID}/rule-groups/{Group}": eval = ac.EvalPermission(ac.ActionAlertingProvisioningWrite) // organization scope + case http.MethodGet + "/api/v1/notifications/time-intervals/{name}", + http.MethodGet + "/api/v1/notifications/time-intervals": + eval = ac.EvalAny(ac.EvalPermission(ac.ActionAlertingNotificationsRead), ac.EvalPermission(ac.ActionAlertingNotificationsTimeIntervalsRead), ac.EvalPermission(ac.ActionAlertingProvisioningRead)) } if eval != nil { diff --git a/pkg/services/ngalert/api/authorization_test.go b/pkg/services/ngalert/api/authorization_test.go index c1f9c43b748..264c74ba7e3 100644 --- a/pkg/services/ngalert/api/authorization_test.go +++ b/pkg/services/ngalert/api/authorization_test.go @@ -40,7 +40,7 @@ func TestAuthorize(t *testing.T) { } paths[p] = methods } - require.Len(t, paths, 60) + require.Len(t, paths, 62) ac := acmock.New() api := &API{AccessControl: ac} diff --git a/pkg/services/ngalert/api/generated_base_api_notifications.go b/pkg/services/ngalert/api/generated_base_api_notifications.go new file mode 100644 index 00000000000..1423daf23dc --- /dev/null +++ b/pkg/services/ngalert/api/generated_base_api_notifications.go @@ -0,0 +1,62 @@ +/*Package api contains base API implementation of unified alerting + * + *Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + * + *Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them. + */ +package api + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/middleware/requestmeta" + contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" + "github.com/grafana/grafana/pkg/services/ngalert/metrics" + "github.com/grafana/grafana/pkg/web" +) + +type NotificationsApi interface { + RouteNotificationsGetTimeInterval(*contextmodel.ReqContext) response.Response + RouteNotificationsGetTimeIntervals(*contextmodel.ReqContext) response.Response +} + +func (f *NotificationsApiHandler) RouteNotificationsGetTimeInterval(ctx *contextmodel.ReqContext) response.Response { + // Parse Path Parameters + nameParam := web.Params(ctx.Req)[":name"] + return f.handleRouteNotificationsGetTimeInterval(ctx, nameParam) +} +func (f *NotificationsApiHandler) RouteNotificationsGetTimeIntervals(ctx *contextmodel.ReqContext) response.Response { + return f.handleRouteNotificationsGetTimeIntervals(ctx) +} + +func (api *API) RegisterNotificationsApiEndpoints(srv NotificationsApi, m *metrics.API) { + api.RouteRegister.Group("", func(group routing.RouteRegister) { + group.Get( + toMacaronPath("/api/v1/notifications/time-intervals/{name}"), + requestmeta.SetOwner(requestmeta.TeamAlerting), + requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), + api.authorize(http.MethodGet, "/api/v1/notifications/time-intervals/{name}"), + metrics.Instrument( + http.MethodGet, + "/api/v1/notifications/time-intervals/{name}", + api.Hooks.Wrap(srv.RouteNotificationsGetTimeInterval), + m, + ), + ) + group.Get( + toMacaronPath("/api/v1/notifications/time-intervals"), + requestmeta.SetOwner(requestmeta.TeamAlerting), + requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), + api.authorize(http.MethodGet, "/api/v1/notifications/time-intervals"), + metrics.Instrument( + http.MethodGet, + "/api/v1/notifications/time-intervals", + api.Hooks.Wrap(srv.RouteNotificationsGetTimeIntervals), + m, + ), + ) + }, middleware.ReqSignedIn) +} diff --git a/pkg/services/ngalert/api/notifications.go b/pkg/services/ngalert/api/notifications.go new file mode 100644 index 00000000000..5b91c9a246a --- /dev/null +++ b/pkg/services/ngalert/api/notifications.go @@ -0,0 +1,34 @@ +package api + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" +) + +type NotificationsApiHandler struct { + muteTimingService MuteTimingService +} + +func NewNotificationsApi(muteTimingService MuteTimingService) NotificationsApi { + return &NotificationsApiHandler{ + muteTimingService: muteTimingService, + } +} + +func (f *NotificationsApiHandler) handleRouteNotificationsGetTimeInterval(ctx *contextmodel.ReqContext, name string) response.Response { + model, err := f.muteTimingService.GetMuteTiming(ctx.Req.Context(), name, ctx.OrgID) + if err != nil { + return errorToResponse(err) + } + return response.JSON(http.StatusOK, model) // TODO convert to timing interval +} + +func (f *NotificationsApiHandler) handleRouteNotificationsGetTimeIntervals(ctx *contextmodel.ReqContext) response.Response { + model, err := f.muteTimingService.GetMuteTimings(ctx.Req.Context(), ctx.OrgID) + if err != nil { + return errorToResponse(err) + } + return response.JSON(http.StatusOK, model) // TODO convert to timing interval +} diff --git a/pkg/services/ngalert/api/tooling/api.json b/pkg/services/ngalert/api/tooling/api.json index 0a9a87eba07..31333220b78 100644 --- a/pkg/services/ngalert/api/tooling/api.json +++ b/pkg/services/ngalert/api/tooling/api.json @@ -1256,6 +1256,15 @@ }, "typeVersion": { "$ref": "#/definitions/FrameTypeVersion" + }, + "uniqueRowIdFields": { + "description": "Array of field indices which values create a unique id for each row. Ideally this should be globally unique ID\nbut that isn't guarantied. Should help with keeping track and deduplicating rows in visualizations, especially\nwith streaming data with frequent updates.", + "example": "TraceID in Tempo, table name + primary key in SQL", + "items": { + "format": "int64", + "type": "integer" + }, + "type": "array" } }, "title": "FrameMeta matches:", @@ -1667,6 +1676,23 @@ ], "type": "object" }, + "GettableTimeIntervals": { + "properties": { + "name": { + "type": "string" + }, + "provenance": { + "$ref": "#/definitions/Provenance" + }, + "time_intervals": { + "items": { + "$ref": "#/definitions/TimeIntervalItem" + }, + "type": "array" + } + }, + "type": "object" + }, "GettableUserConfig": { "properties": { "alertmanager_config": { @@ -2816,6 +2842,20 @@ }, "type": "object" }, + "PostableTimeIntervals": { + "properties": { + "name": { + "type": "string" + }, + "time_intervals": { + "items": { + "$ref": "#/definitions/TimeIntervalItem" + }, + "type": "array" + } + }, + "type": "object" + }, "PostableUserConfig": { "properties": { "alertmanager_config": { @@ -4166,6 +4206,55 @@ }, "type": "object" }, + "TimeIntervalItem": { + "properties": { + "days_of_month": { + "items": { + "type": "string" + }, + "type": "array" + }, + "location": { + "type": "string" + }, + "months": { + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "items": { + "$ref": "#/definitions/TimeIntervalTimeRange" + }, + "type": "array" + }, + "weekdays": { + "items": { + "type": "string" + }, + "type": "array" + }, + "years": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "TimeIntervalTimeRange": { + "properties": { + "end_time": { + "type": "string" + }, + "start_time": { + "type": "string" + } + }, + "type": "object" + }, "TimeRange": { "description": "Redefining this to avoid an import cycle", "properties": { @@ -4181,7 +4270,6 @@ "type": "object" }, "URL": { - "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use the EscapedPath method, which preserves\nthe original encoding of Path.\n\nThe RawPath field is an optional field which is only set when the default\nencoding of Path is different from the escaped path. See the EscapedPath method\nfor more details.\n\nURL's String method uses the EscapedPath method to obtain the path.", "properties": { "ForceQuery": { "type": "boolean" @@ -4217,7 +4305,7 @@ "$ref": "#/definitions/Userinfo" } }, - "title": "A URL represents a parsed URL (technically, a URI reference).", + "title": "URL is a custom URL type that allows validation at configuration load time.", "type": "object" }, "UpdateRuleGroupResponse": { @@ -4613,6 +4701,7 @@ "type": "array" }, "gettableSilence": { + "description": "GettableSilence gettable silence", "properties": { "comment": { "description": "comment", @@ -4661,13 +4750,13 @@ "type": "object" }, "gettableSilences": { - "description": "GettableSilences gettable silences", "items": { "$ref": "#/definitions/gettableSilence" }, "type": "array" }, "integration": { + "description": "Integration integration", "properties": { "lastNotifyAttempt": { "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time", @@ -4849,7 +4938,6 @@ "type": "object" }, "receiver": { - "description": "Receiver receiver", "properties": { "active": { "description": "active", @@ -4969,6 +5057,61 @@ "version": "1.1.0" }, "paths": { + "/v1/notifications/time-intervals": { + "get": { + "description": "Get all the time intervals", + "operationId": "RouteNotificationsGetTimeIntervals", + "responses": { + "200": { + "$ref": "#/responses/GetAllIntervalsResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + } + }, + "tags": [ + "notifications" + ] + } + }, + "/v1/notifications/time-intervals/{name}": { + "get": { + "operationId": "RouteNotificationsGetTimeInterval", + "parameters": [ + { + "description": "Time interval name", + "in": "path", + "name": "name", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "$ref": "#/responses/GetIntervalsByNameResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + }, + "404": { + "description": "NotFound", + "schema": { + "$ref": "#/definitions/NotFound" + } + } + }, + "summary": "Get a time interval by name.", + "tags": [ + "notifications" + ] + } + }, "/v1/provisioning/alert-rules": { "get": { "operationId": "RouteGetAlertRules", @@ -6010,6 +6153,21 @@ "application/json" ], "responses": { + "GetAllIntervalsResponse": { + "description": "", + "schema": { + "items": { + "$ref": "#/definitions/GettableTimeIntervals" + }, + "type": "array" + } + }, + "GetIntervalsByNameResponse": { + "description": "", + "schema": { + "$ref": "#/definitions/GettableTimeIntervals" + } + }, "GettableHistoricUserConfigs": { "description": "", "schema": { diff --git a/pkg/services/ngalert/api/tooling/definitions/time_intervals.go b/pkg/services/ngalert/api/tooling/definitions/time_intervals.go new file mode 100644 index 00000000000..04ac68aa845 --- /dev/null +++ b/pkg/services/ngalert/api/tooling/definitions/time_intervals.go @@ -0,0 +1,64 @@ +package definitions + +// swagger:route GET /v1/notifications/time-intervals notifications stable RouteNotificationsGetTimeIntervals +// +// Get all the time intervals +// +// Responses: +// 200: GetAllIntervalsResponse +// 403: ForbiddenError + +// swagger:route GET /v1/notifications/time-intervals/{name} notifications stable RouteNotificationsGetTimeInterval +// +// Get a time interval by name. +// +// Responses: +// 200: GetIntervalsByNameResponse +// 404: NotFound +// 403: ForbiddenError + +// swagger:parameters stable RouteNotificationsGetTimeInterval +type RouteTimeIntervalNameParam struct { + // Time interval name + // in:path + Name string `json:"name"` +} + +// swagger:response GetAllIntervalsResponse +type GetAllIntervalsResponse struct { + // in:body + Body []GettableTimeIntervals +} + +// swagger:response GetIntervalsByNameResponse +type GetIntervalsByNameResponse struct { + // in:body + Body GettableTimeIntervals +} + +// swagger:model +type PostableTimeIntervals struct { + Name string `json:"name" hcl:"name"` + TimeIntervals []TimeIntervalItem `json:"time_intervals" hcl:"intervals,block"` +} + +type TimeIntervalItem struct { + Times []TimeIntervalTimeRange `json:"times,omitempty" hcl:"times,block"` + Weekdays *[]string `json:"weekdays,omitempty" hcl:"weekdays"` + DaysOfMonth *[]string `json:"days_of_month,omitempty" hcl:"days_of_month"` + Months *[]string `json:"months,omitempty" hcl:"months"` + Years *[]string `json:"years,omitempty" hcl:"years"` + Location *string `json:"location,omitempty" hcl:"location"` +} + +type TimeIntervalTimeRange struct { + StartMinute string `json:"start_time" hcl:"start"` + EndMinute string `json:"end_time" hcl:"end"` +} + +// swagger:model +type GettableTimeIntervals struct { + Name string `json:"name" hcl:"name"` + TimeIntervals []TimeIntervalItem `json:"time_intervals" hcl:"intervals,block"` + Provenance Provenance `json:"provenance,omitempty"` +} diff --git a/pkg/services/ngalert/api/tooling/post.json b/pkg/services/ngalert/api/tooling/post.json index c3d72b28d26..cfb69720ed1 100644 --- a/pkg/services/ngalert/api/tooling/post.json +++ b/pkg/services/ngalert/api/tooling/post.json @@ -1256,6 +1256,15 @@ }, "typeVersion": { "$ref": "#/definitions/FrameTypeVersion" + }, + "uniqueRowIdFields": { + "description": "Array of field indices which values create a unique id for each row. Ideally this should be globally unique ID\nbut that isn't guarantied. Should help with keeping track and deduplicating rows in visualizations, especially\nwith streaming data with frequent updates.", + "example": "TraceID in Tempo, table name + primary key in SQL", + "items": { + "format": "int64", + "type": "integer" + }, + "type": "array" } }, "title": "FrameMeta matches:", @@ -1667,6 +1676,23 @@ ], "type": "object" }, + "GettableTimeIntervals": { + "properties": { + "name": { + "type": "string" + }, + "provenance": { + "$ref": "#/definitions/Provenance" + }, + "time_intervals": { + "items": { + "$ref": "#/definitions/TimeIntervalItem" + }, + "type": "array" + } + }, + "type": "object" + }, "GettableUserConfig": { "properties": { "alertmanager_config": { @@ -2816,6 +2842,20 @@ }, "type": "object" }, + "PostableTimeIntervals": { + "properties": { + "name": { + "type": "string" + }, + "time_intervals": { + "items": { + "$ref": "#/definitions/TimeIntervalItem" + }, + "type": "array" + } + }, + "type": "object" + }, "PostableUserConfig": { "properties": { "alertmanager_config": { @@ -4166,6 +4206,55 @@ }, "type": "object" }, + "TimeIntervalItem": { + "properties": { + "days_of_month": { + "items": { + "type": "string" + }, + "type": "array" + }, + "location": { + "type": "string" + }, + "months": { + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "items": { + "$ref": "#/definitions/TimeIntervalTimeRange" + }, + "type": "array" + }, + "weekdays": { + "items": { + "type": "string" + }, + "type": "array" + }, + "years": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "TimeIntervalTimeRange": { + "properties": { + "end_time": { + "type": "string" + }, + "start_time": { + "type": "string" + } + }, + "type": "object" + }, "TimeRange": { "description": "Redefining this to avoid an import cycle", "properties": { @@ -4181,6 +4270,7 @@ "type": "object" }, "URL": { + "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use the EscapedPath method, which preserves\nthe original encoding of Path.\n\nThe RawPath field is an optional field which is only set when the default\nencoding of Path is different from the escaped path. See the EscapedPath method\nfor more details.\n\nURL's String method uses the EscapedPath method to obtain the path.", "properties": { "ForceQuery": { "type": "boolean" @@ -4216,7 +4306,7 @@ "$ref": "#/definitions/Userinfo" } }, - "title": "URL is a custom URL type that allows validation at configuration load time.", + "title": "A URL represents a parsed URL (technically, a URI reference).", "type": "object" }, "UpdateRuleGroupResponse": { @@ -4422,6 +4512,7 @@ "type": "object" }, "alertGroup": { + "description": "AlertGroup alert group", "properties": { "alerts": { "description": "alerts", @@ -4445,7 +4536,6 @@ "type": "object" }, "alertGroups": { - "description": "AlertGroups alert groups", "items": { "$ref": "#/definitions/alertGroup" }, @@ -4550,7 +4640,6 @@ "type": "object" }, "gettableAlert": { - "description": "GettableAlert gettable alert", "properties": { "annotations": { "$ref": "#/definitions/labelSet" @@ -4612,6 +4701,7 @@ "type": "array" }, "gettableSilence": { + "description": "GettableSilence gettable silence", "properties": { "comment": { "description": "comment", @@ -4660,14 +4750,12 @@ "type": "object" }, "gettableSilences": { - "description": "GettableSilences gettable silences", "items": { "$ref": "#/definitions/gettableSilence" }, "type": "array" }, "integration": { - "description": "Integration integration", "properties": { "lastNotifyAttempt": { "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time", @@ -4811,7 +4899,6 @@ "type": "array" }, "postableSilence": { - "description": "PostableSilence postable silence", "properties": { "comment": { "description": "comment", @@ -4849,6 +4936,7 @@ "type": "object" }, "receiver": { + "description": "Receiver receiver", "properties": { "active": { "description": "active", @@ -6958,6 +7046,61 @@ ] } }, + "/v1/notifications/time-intervals": { + "get": { + "description": "Get all the time intervals", + "operationId": "RouteNotificationsGetTimeIntervals", + "responses": { + "200": { + "$ref": "#/responses/GetAllIntervalsResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + } + }, + "tags": [ + "notifications" + ] + } + }, + "/v1/notifications/time-intervals/{name}": { + "get": { + "operationId": "RouteNotificationsGetTimeInterval", + "parameters": [ + { + "description": "Time interval name", + "in": "path", + "name": "name", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "$ref": "#/responses/GetIntervalsByNameResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + }, + "404": { + "description": "NotFound", + "schema": { + "$ref": "#/definitions/NotFound" + } + } + }, + "summary": "Get a time interval by name.", + "tags": [ + "notifications" + ] + } + }, "/v1/provisioning/alert-rules": { "get": { "operationId": "RouteGetAlertRules", @@ -8349,6 +8492,21 @@ "application/json" ], "responses": { + "GetAllIntervalsResponse": { + "description": "", + "schema": { + "items": { + "$ref": "#/definitions/GettableTimeIntervals" + }, + "type": "array" + } + }, + "GetIntervalsByNameResponse": { + "description": "", + "schema": { + "$ref": "#/definitions/GettableTimeIntervals" + } + }, "GettableHistoricUserConfigs": { "description": "", "schema": { diff --git a/pkg/services/ngalert/api/tooling/spec.json b/pkg/services/ngalert/api/tooling/spec.json index a67c47a68a0..50e830b39da 100644 --- a/pkg/services/ngalert/api/tooling/spec.json +++ b/pkg/services/ngalert/api/tooling/spec.json @@ -2007,6 +2007,63 @@ } } }, + "/v1/notifications/time-intervals": { + "get": { + "description": "Get all the time intervals", + "tags": [ + "notifications", + "stable" + ], + "operationId": "RouteNotificationsGetTimeIntervals", + "responses": { + "200": { + "$ref": "#/responses/GetAllIntervalsResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + } + } + } + }, + "/v1/notifications/time-intervals/{name}": { + "get": { + "tags": [ + "notifications", + "stable" + ], + "summary": "Get a time interval by name.", + "operationId": "RouteNotificationsGetTimeInterval", + "parameters": [ + { + "type": "string", + "description": "Time interval name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/GetIntervalsByNameResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + }, + "404": { + "description": "NotFound", + "schema": { + "$ref": "#/definitions/NotFound" + } + } + } + } + }, "/v1/provisioning/alert-rules": { "get": { "tags": [ @@ -4682,6 +4739,15 @@ }, "typeVersion": { "$ref": "#/definitions/FrameTypeVersion" + }, + "uniqueRowIdFields": { + "description": "Array of field indices which values create a unique id for each row. Ideally this should be globally unique ID\nbut that isn't guarantied. Should help with keeping track and deduplicating rows in visualizations, especially\nwith streaming data with frequent updates.", + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "example": "TraceID in Tempo, table name + primary key in SQL" } } }, @@ -5091,6 +5157,23 @@ } } }, + "GettableTimeIntervals": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "provenance": { + "$ref": "#/definitions/Provenance" + }, + "time_intervals": { + "type": "array", + "items": { + "$ref": "#/definitions/TimeIntervalItem" + } + } + } + }, "GettableUserConfig": { "type": "object", "properties": { @@ -6241,6 +6324,20 @@ } } }, + "PostableTimeIntervals": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "time_intervals": { + "type": "array", + "items": { + "$ref": "#/definitions/TimeIntervalItem" + } + } + } + }, "PostableUserConfig": { "type": "object", "properties": { @@ -7591,6 +7688,55 @@ } } }, + "TimeIntervalItem": { + "type": "object", + "properties": { + "days_of_month": { + "type": "array", + "items": { + "type": "string" + } + }, + "location": { + "type": "string" + }, + "months": { + "type": "array", + "items": { + "type": "string" + } + }, + "times": { + "type": "array", + "items": { + "$ref": "#/definitions/TimeIntervalTimeRange" + } + }, + "weekdays": { + "type": "array", + "items": { + "type": "string" + } + }, + "years": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "TimeIntervalTimeRange": { + "type": "object", + "properties": { + "end_time": { + "type": "string" + }, + "start_time": { + "type": "string" + } + } + }, "TimeRange": { "description": "Redefining this to avoid an import cycle", "type": "object", @@ -7606,8 +7752,9 @@ } }, "URL": { + "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use the EscapedPath method, which preserves\nthe original encoding of Path.\n\nThe RawPath field is an optional field which is only set when the default\nencoding of Path is different from the escaped path. See the EscapedPath method\nfor more details.\n\nURL's String method uses the EscapedPath method to obtain the path.", "type": "object", - "title": "URL is a custom URL type that allows validation at configuration load time.", + "title": "A URL represents a parsed URL (technically, a URI reference).", "properties": { "ForceQuery": { "type": "boolean" @@ -7847,6 +7994,7 @@ } }, "alertGroup": { + "description": "AlertGroup alert group", "type": "object", "required": [ "alerts", @@ -7871,7 +8019,6 @@ "$ref": "#/definitions/alertGroup" }, "alertGroups": { - "description": "AlertGroups alert groups", "type": "array", "items": { "$ref": "#/definitions/alertGroup" @@ -7977,7 +8124,6 @@ } }, "gettableAlert": { - "description": "GettableAlert gettable alert", "type": "object", "required": [ "labels", @@ -8041,6 +8187,7 @@ "$ref": "#/definitions/gettableAlerts" }, "gettableSilence": { + "description": "GettableSilence gettable silence", "type": "object", "required": [ "comment", @@ -8090,7 +8237,6 @@ "$ref": "#/definitions/gettableSilence" }, "gettableSilences": { - "description": "GettableSilences gettable silences", "type": "array", "items": { "$ref": "#/definitions/gettableSilence" @@ -8098,7 +8244,6 @@ "$ref": "#/definitions/gettableSilences" }, "integration": { - "description": "Integration integration", "type": "object", "required": [ "name", @@ -8243,7 +8388,6 @@ } }, "postableSilence": { - "description": "PostableSilence postable silence", "type": "object", "required": [ "comment", @@ -8282,6 +8426,7 @@ "$ref": "#/definitions/postableSilence" }, "receiver": { + "description": "Receiver receiver", "type": "object", "required": [ "active", @@ -8397,6 +8542,21 @@ } }, "responses": { + "GetAllIntervalsResponse": { + "description": "", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/GettableTimeIntervals" + } + } + }, + "GetIntervalsByNameResponse": { + "description": "", + "schema": { + "$ref": "#/definitions/GettableTimeIntervals" + } + }, "GettableHistoricUserConfigs": { "description": "", "schema": { diff --git a/pkg/tests/api/alerting/api_notifications_time_interval_test.go b/pkg/tests/api/alerting/api_notifications_time_interval_test.go new file mode 100644 index 00000000000..76b890ecdc5 --- /dev/null +++ b/pkg/tests/api/alerting/api_notifications_time_interval_test.go @@ -0,0 +1,204 @@ +package alerting + +import ( + "net/http" + "slices" + "strings" + "testing" + "time" + + "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/timeinterval" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/tests/testinfra" + "github.com/grafana/grafana/pkg/util" +) + +func TestTimeInterval(t *testing.T) { + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ + DisableLegacyAlerting: true, + EnableUnifiedAlerting: true, + DisableAnonymous: true, + AppModeProduction: true, + }) + + grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path) + + createUser(t, store, user.CreateUserCommand{ + DefaultOrgRole: string(org.RoleAdmin), + Password: "admin", + Login: "admin", + }) + + apiClient := newAlertingApiClient(grafanaListedAddr, "admin", "admin") + + t.Run("default config should return empty list", func(t *testing.T) { + mt, status, body := apiClient.GetAllTimeIntervalsWithStatus(t) + requireStatusCode(t, http.StatusOK, status, body) + require.Empty(t, mt) + }) + + emptyTimeInterval := definitions.PostableTimeIntervals{ + Name: "Empty Mute Timing", + TimeIntervals: []definitions.TimeIntervalItem{}, + } + + func() { + // TODO replace with Time-Interval later + emptyMuteTiming := definitions.MuteTimeInterval{ + MuteTimeInterval: config.MuteTimeInterval{ + Name: "Empty Mute Timing", + TimeIntervals: []timeinterval.TimeInterval{}, + }, + } + + // TODO replace with create interval API + // t.Run("should create a new mute timing without any intervals", func(t *testing.T) { + mt, status, body := apiClient.CreateMuteTimingWithStatus(t, emptyMuteTiming) + requireStatusCode(t, http.StatusCreated, status, body) + require.Equal(t, emptyMuteTiming.MuteTimeInterval, mt.MuteTimeInterval) + require.EqualValues(t, models.ProvenanceAPI, mt.Provenance) + // }) + + anotherMuteTiming := definitions.MuteTimeInterval{ + MuteTimeInterval: config.MuteTimeInterval{ + Name: "Not Empty Mute Timing", + TimeIntervals: []timeinterval.TimeInterval{ + { + Times: []timeinterval.TimeRange{ + { + StartMinute: 10, + EndMinute: 45, + }, + }, + Weekdays: []timeinterval.WeekdayRange{ + { + InclusiveRange: timeinterval.InclusiveRange{ + Begin: 0, + End: 2, + }, + }, + { + InclusiveRange: timeinterval.InclusiveRange{ + Begin: 4, + End: 5, + }, + }, + }, + DaysOfMonth: []timeinterval.DayOfMonthRange{ + { + InclusiveRange: timeinterval.InclusiveRange{ + Begin: 1, + End: 7, + }, + }, + { + InclusiveRange: timeinterval.InclusiveRange{ + Begin: 14, + End: 28, + }, + }, + }, + Months: []timeinterval.MonthRange{ + { + InclusiveRange: timeinterval.InclusiveRange{ + Begin: 1, + End: 5, + }, + }, + }, + Years: []timeinterval.YearRange{ + { + InclusiveRange: timeinterval.InclusiveRange{ + Begin: 2024, + End: 2025, + }, + }, + }, + Location: &timeinterval.Location{ + Location: time.UTC, + }, + }, + }, + }, + } + + // t.Run("should create a new mute timing with some settings", func(t *testing.T) { + mt, status, body = apiClient.CreateMuteTimingWithStatus(t, anotherMuteTiming) + requireStatusCode(t, http.StatusCreated, status, body) + require.Equal(t, anotherMuteTiming.MuteTimeInterval, mt.MuteTimeInterval) + require.EqualValues(t, models.ProvenanceAPI, mt.Provenance) + // }) + }() + + anotherTimeInterval := definitions.PostableTimeIntervals{ + Name: "Not Empty Mute Timing", + TimeIntervals: []definitions.TimeIntervalItem{ + { + Times: []definitions.TimeIntervalTimeRange{ + { + StartMinute: "00:10", + EndMinute: "00:45", + }, + }, + Weekdays: util.Pointer([]string{ + "sunday:tuesday", + "thursday:friday", + }), + DaysOfMonth: util.Pointer([]string{ + "1:7", + "14:28", + }), + Months: util.Pointer([]string{ + "1:5", + }), + Years: util.Pointer([]string{ + "2024:2025", + }), + Location: util.Pointer("UTC"), + }, + }, + } + + t.Run("should return time interval by name", func(t *testing.T) { + ti, status, body := apiClient.GetTimeIntervalByNameWithStatus(t, emptyTimeInterval.Name) + requireStatusCode(t, http.StatusOK, status, body) + require.Equal(t, emptyTimeInterval.TimeIntervals, ti.TimeIntervals) + require.Equal(t, emptyTimeInterval.Name, ti.Name) + require.EqualValues(t, models.ProvenanceAPI, ti.Provenance) + + ti, status, body = apiClient.GetTimeIntervalByNameWithStatus(t, anotherTimeInterval.Name) + requireStatusCode(t, http.StatusOK, status, body) + require.Equal(t, anotherTimeInterval.TimeIntervals, ti.TimeIntervals) + require.Equal(t, anotherTimeInterval.Name, ti.Name) + require.EqualValues(t, models.ProvenanceAPI, ti.Provenance) + }) + + t.Run("should return NotFound if time interval does not exist", func(t *testing.T) { + _, status, body := apiClient.GetTimeIntervalByNameWithStatus(t, "some-missing-timing") + requireStatusCode(t, http.StatusNotFound, status, body) + }) + + t.Run("should return all mute timings", func(t *testing.T) { + mt, status, body := apiClient.GetAllTimeIntervalsWithStatus(t) + requireStatusCode(t, http.StatusOK, status, body) + require.Len(t, mt, 2) + + slices.SortFunc(mt, func(a, b definitions.GettableTimeIntervals) int { + return strings.Compare(a.Name, b.Name) + }) + + require.Equal(t, emptyTimeInterval.TimeIntervals, mt[0].TimeIntervals) + require.Equal(t, emptyTimeInterval.Name, mt[0].Name) + require.EqualValues(t, models.ProvenanceAPI, mt[0].Provenance) + + require.Equal(t, anotherTimeInterval.TimeIntervals, mt[1].TimeIntervals) + require.Equal(t, anotherTimeInterval.Name, mt[1].Name) + require.EqualValues(t, models.ProvenanceAPI, mt[1].Provenance) + }) +} diff --git a/pkg/tests/api/alerting/testing.go b/pkg/tests/api/alerting/testing.go index 7d3bb10aed8..0a3ab45e00b 100644 --- a/pkg/tests/api/alerting/testing.go +++ b/pkg/tests/api/alerting/testing.go @@ -756,6 +756,24 @@ func (a apiClient) GetRuleHistoryWithStatus(t *testing.T, ruleUID string) (data. return sendRequest[data.Frame](t, req, http.StatusOK) } +func (a apiClient) GetAllTimeIntervalsWithStatus(t *testing.T) ([]apimodels.GettableTimeIntervals, int, string) { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/notifications/time-intervals", a.url), nil) + require.NoError(t, err) + + return sendRequest[[]apimodels.GettableTimeIntervals](t, req, http.StatusOK) +} + +func (a apiClient) GetTimeIntervalByNameWithStatus(t *testing.T, name string) (apimodels.GettableTimeIntervals, int, string) { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/v1/notifications/time-intervals/%s", a.url, name), nil) + require.NoError(t, err) + + return sendRequest[apimodels.GettableTimeIntervals](t, req, http.StatusOK) +} + func sendRequest[T any](t *testing.T, req *http.Request, successStatusCode int) (T, int, string) { client := &http.Client{} resp, err := client.Do(req) diff --git a/public/api-merged.json b/public/api-merged.json index 2293bfa87c5..7fb4e1c5b67 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -10362,6 +10362,61 @@ } } }, + "/v1/notifications/time-intervals": { + "get": { + "description": "Get all the time intervals", + "tags": [ + "notifications" + ], + "operationId": "RouteNotificationsGetTimeIntervals", + "responses": { + "200": { + "$ref": "#/responses/GetAllIntervalsResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + } + } + } + }, + "/v1/notifications/time-intervals/{name}": { + "get": { + "tags": [ + "notifications" + ], + "summary": "Get a time interval by name.", + "operationId": "RouteNotificationsGetTimeInterval", + "parameters": [ + { + "type": "string", + "description": "Time interval name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/GetIntervalsByNameResponse" + }, + "403": { + "description": "ForbiddenError", + "schema": { + "$ref": "#/definitions/ForbiddenError" + } + }, + "404": { + "description": "NotFound", + "schema": { + "$ref": "#/definitions/NotFound" + } + } + } + } + }, "/v1/provisioning/alert-rules": { "get": { "tags": [ @@ -15533,6 +15588,23 @@ } } }, + "GettableTimeIntervals": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "provenance": { + "$ref": "#/definitions/Provenance" + }, + "time_intervals": { + "type": "array", + "items": { + "$ref": "#/definitions/TimeIntervalItem" + } + } + } + }, "GettableUserConfig": { "type": "object", "properties": { @@ -17781,6 +17853,20 @@ } } }, + "PostableTimeIntervals": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "time_intervals": { + "type": "array", + "items": { + "$ref": "#/definitions/TimeIntervalItem" + } + } + } + }, "PostableUserConfig": { "type": "object", "properties": { @@ -20448,6 +20534,55 @@ } } }, + "TimeIntervalItem": { + "type": "object", + "properties": { + "days_of_month": { + "type": "array", + "items": { + "type": "string" + } + }, + "location": { + "type": "string" + }, + "months": { + "type": "array", + "items": { + "type": "string" + } + }, + "times": { + "type": "array", + "items": { + "$ref": "#/definitions/TimeIntervalTimeRange" + } + }, + "weekdays": { + "type": "array", + "items": { + "type": "string" + } + }, + "years": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "TimeIntervalTimeRange": { + "type": "object", + "properties": { + "end_time": { + "type": "string" + }, + "start_time": { + "type": "string" + } + } + }, "TimeRange": { "description": "Redefining this to avoid an import cycle", "type": "object", @@ -21719,6 +21854,7 @@ } }, "gettableSilence": { + "description": "GettableSilence gettable silence", "type": "object", "required": [ "comment", @@ -21767,13 +21903,13 @@ } }, "gettableSilences": { - "description": "GettableSilences gettable silences", "type": "array", "items": { "$ref": "#/definitions/gettableSilence" } }, "integration": { + "description": "Integration integration", "type": "object", "required": [ "name", @@ -21983,7 +22119,6 @@ } }, "receiver": { - "description": "Receiver receiver", "type": "object", "required": [ "active", @@ -22170,6 +22305,21 @@ } }, "responses": { + "GetAllIntervalsResponse": { + "description": "(empty)", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/GettableTimeIntervals" + } + } + }, + "GetIntervalsByNameResponse": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/GettableTimeIntervals" + } + }, "GettableHistoricUserConfigs": { "description": "(empty)", "schema": { diff --git a/public/openapi3.json b/public/openapi3.json index d695f4645dd..fd6a44c65d6 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -1,6 +1,29 @@ { "components": { "responses": { + "GetAllIntervalsResponse": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/GettableTimeIntervals" + }, + "type": "array" + } + } + }, + "description": "(empty)" + }, + "GetIntervalsByNameResponse": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GettableTimeIntervals" + } + } + }, + "description": "(empty)" + }, "GettableHistoricUserConfigs": { "content": { "application/json": { @@ -6049,6 +6072,23 @@ ], "type": "object" }, + "GettableTimeIntervals": { + "properties": { + "name": { + "type": "string" + }, + "provenance": { + "$ref": "#/components/schemas/Provenance" + }, + "time_intervals": { + "items": { + "$ref": "#/components/schemas/TimeIntervalItem" + }, + "type": "array" + } + }, + "type": "object" + }, "GettableUserConfig": { "properties": { "alertmanager_config": { @@ -8297,6 +8337,20 @@ }, "type": "object" }, + "PostableTimeIntervals": { + "properties": { + "name": { + "type": "string" + }, + "time_intervals": { + "items": { + "$ref": "#/components/schemas/TimeIntervalItem" + }, + "type": "array" + } + }, + "type": "object" + }, "PostableUserConfig": { "properties": { "alertmanager_config": { @@ -10963,6 +11017,55 @@ }, "type": "object" }, + "TimeIntervalItem": { + "properties": { + "days_of_month": { + "items": { + "type": "string" + }, + "type": "array" + }, + "location": { + "type": "string" + }, + "months": { + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "items": { + "$ref": "#/components/schemas/TimeIntervalTimeRange" + }, + "type": "array" + }, + "weekdays": { + "items": { + "type": "string" + }, + "type": "array" + }, + "years": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "TimeIntervalTimeRange": { + "properties": { + "end_time": { + "type": "string" + }, + "start_time": { + "type": "string" + } + }, + "type": "object" + }, "TimeRange": { "description": "Redefining this to avoid an import cycle", "properties": { @@ -12234,6 +12337,7 @@ "type": "array" }, "gettableSilence": { + "description": "GettableSilence gettable silence", "properties": { "comment": { "description": "comment", @@ -12282,13 +12386,13 @@ "type": "object" }, "gettableSilences": { - "description": "GettableSilences gettable silences", "items": { "$ref": "#/components/schemas/gettableSilence" }, "type": "array" }, "integration": { + "description": "Integration integration", "properties": { "lastNotifyAttempt": { "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time", @@ -12498,7 +12602,6 @@ "type": "object" }, "receiver": { - "description": "Receiver receiver", "properties": { "active": { "description": "active", @@ -23804,6 +23907,75 @@ ] } }, + "/v1/notifications/time-intervals": { + "get": { + "description": "Get all the time intervals", + "operationId": "RouteNotificationsGetTimeIntervals", + "responses": { + "200": { + "$ref": "#/components/responses/GetAllIntervalsResponse" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ForbiddenError" + } + } + }, + "description": "ForbiddenError" + } + }, + "tags": [ + "notifications" + ] + } + }, + "/v1/notifications/time-intervals/{name}": { + "get": { + "operationId": "RouteNotificationsGetTimeInterval", + "parameters": [ + { + "description": "Time interval name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/GetIntervalsByNameResponse" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ForbiddenError" + } + } + }, + "description": "ForbiddenError" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFound" + } + } + }, + "description": "NotFound" + } + }, + "summary": "Get a time interval by name.", + "tags": [ + "notifications" + ] + } + }, "/v1/provisioning/alert-rules": { "get": { "operationId": "RouteGetAlertRules", From 7a17963ab98b07cb72ba2f61009cbf9ff829fcff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 1 Feb 2024 23:10:15 +0100 Subject: [PATCH 73/84] Scenes: Update to 2.4 (#81723) * Scenes: Update to 2.4 --- package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 5d90f6a5f7d..fb28910137d 100644 --- a/package.json +++ b/package.json @@ -245,7 +245,7 @@ "@grafana/monaco-logql": "^0.0.7", "@grafana/o11y-ds-frontend": "workspace:*", "@grafana/runtime": "workspace:*", - "@grafana/scenes": "^2.2.0", + "@grafana/scenes": "^2.4.0", "@grafana/schema": "workspace:*", "@grafana/sql": "workspace:*", "@grafana/ui": "workspace:*", diff --git a/yarn.lock b/yarn.lock index b0e6f56209b..c3a56996a52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3731,9 +3731,9 @@ __metadata: languageName: unknown linkType: soft -"@grafana/scenes@npm:^2.2.0": - version: 2.2.2 - resolution: "@grafana/scenes@npm:2.2.2" +"@grafana/scenes@npm:^2.4.0": + version: 2.4.0 + resolution: "@grafana/scenes@npm:2.4.0" dependencies: "@grafana/e2e-selectors": "npm:10.0.2" react-grid-layout: "npm:1.3.4" @@ -3745,7 +3745,7 @@ __metadata: "@grafana/runtime": 10.0.3 "@grafana/schema": 10.0.3 "@grafana/ui": 10.0.3 - checksum: 10/91f17fb1c60e4e355d9f1846412febd138aee27d4713bd76ccc4fa80a8811f7038862d3eb6a52fe721db014e343b265110786433d64d547c0b51ebdbefcff889 + checksum: 10/14046fdf94cf84282599a682992cbadb86e951bfd5969283b97d3cd3ac8cc81266cf50b881d76fe0eb51193ed1edc097df8ad8b7f7c9b94ff5041a3704dd8438 languageName: node linkType: hard @@ -17622,7 +17622,7 @@ __metadata: "@grafana/monaco-logql": "npm:^0.0.7" "@grafana/o11y-ds-frontend": "workspace:*" "@grafana/runtime": "workspace:*" - "@grafana/scenes": "npm:^2.2.0" + "@grafana/scenes": "npm:^2.4.0" "@grafana/schema": "workspace:*" "@grafana/sql": "workspace:*" "@grafana/tsconfig": "npm:^1.3.0-rc1" From 67b6be5515f01a88e58fe0fcad8d9eadc7149625 Mon Sep 17 00:00:00 2001 From: Todd Treece <360020+toddtreece@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:27:30 -0500 Subject: [PATCH 74/84] K8s: Refactor config/options for aggregation (#81739) --- .github/CODEOWNERS | 3 +- pkg/aggregator/aggregator.go | 628 ------------------ pkg/aggregator/config.go | 57 -- pkg/aggregator/crdRegistrationController.go | 214 ------ pkg/api/frontendsettings_test.go | 2 +- pkg/api/http_server.go | 4 +- pkg/api/metrics.go | 2 +- pkg/api/playlist.go | 4 +- pkg/cmd/grafana/apiserver/aggregator.md | 51 -- pkg/cmd/grafana/apiserver/cmd.go | 109 +-- pkg/cmd/grafana/apiserver/server.go | 13 +- pkg/cmd/grafana/main.go | 12 - pkg/registry/apis/apis.go | 2 + .../apis/dashboard/access/sql_dashboards.go | 4 +- pkg/registry/apis/dashboard/authorizer.go | 2 +- pkg/registry/apis/dashboard/legacy_storage.go | 2 +- pkg/registry/apis/dashboard/register.go | 19 +- pkg/registry/apis/dashboard/sub_access.go | 2 +- pkg/registry/apis/dashboard/sub_versions.go | 2 +- .../apis/dashboard/summary_storage.go | 2 +- pkg/registry/apis/datasource/plugincontext.go | 4 +- pkg/registry/apis/datasource/querier.go | 148 +++++ pkg/registry/apis/datasource/register.go | 11 +- pkg/registry/apis/datasource/standalone.go | 2 +- pkg/registry/apis/example/dummy_storage.go | 4 +- pkg/registry/apis/example/register.go | 17 +- pkg/registry/apis/example/storage.go | 2 +- pkg/registry/apis/example/subresource.go | 2 +- pkg/registry/apis/featuretoggle/features.go | 2 +- pkg/registry/apis/featuretoggle/register.go | 15 +- pkg/registry/apis/featuretoggle/toggles.go | 2 +- pkg/registry/apis/folders/conversions.go | 4 +- pkg/registry/apis/folders/legacy_storage.go | 6 +- pkg/registry/apis/folders/register.go | 17 +- pkg/registry/apis/folders/storage.go | 4 +- pkg/registry/apis/folders/sub_children.go | 2 +- pkg/registry/apis/folders/sub_parents.go | 2 +- pkg/registry/apis/playlist/conversions.go | 4 +- .../apis/playlist/conversions_test.go | 2 +- pkg/registry/apis/playlist/legacy_storage.go | 2 +- pkg/registry/apis/playlist/register.go | 15 +- pkg/registry/apis/playlist/storage.go | 4 +- pkg/registry/apis/query/register.go | 15 +- pkg/registry/apis/service/register.go | 12 +- pkg/registry/apis/service/storage.go | 6 +- pkg/registry/apis/wireset.go | 2 + .../backgroundsvcs/background_services.go | 2 +- pkg/server/wire.go | 2 +- .../README.md | 0 .../apiserver}/aggregator/README.md | 0 .../apiserver/aggregator/aggregator.go | 285 ++++++++ .../aggregator/availableController.go | 0 .../apiserver}/aggregator/resolver.go | 0 .../auth/authorizer/impersonation.go | 0 .../auth/authorizer/impersonation_test.go | 0 .../auth/authorizer/org_id.go | 2 +- .../auth/authorizer/org_role.go | 0 .../auth/authorizer/provider.go | 0 .../auth/authorizer/stack_id.go | 2 +- .../builder}/common.go | 7 +- .../builder}/helper.go | 62 +- .../builder}/openapi.go | 6 +- .../builder}/request_handler.go | 8 +- pkg/services/apiserver/config.go | 48 ++ .../endpoints/request/namespace.go | 0 .../endpoints/request/namespace_test.go | 2 +- pkg/services/apiserver/options/aggregator.go | 112 ++++ pkg/services/apiserver/options/extra.go | 46 ++ .../options}/log.go | 2 +- pkg/services/apiserver/options/options.go | 126 ++++ pkg/services/apiserver/options/storage.go | 51 ++ .../registry/generic/strategy.go | 0 .../rest/dualwriter.go | 0 .../service.go | 231 +++---- .../storage/entity/fieldRequirements.go | 0 .../storage/entity/restoptions.go | 0 .../storage/entity/storage.go | 0 .../storage/entity/utils.go | 2 +- .../storage/entity/utils_test.go | 0 .../storage/file/file.go | 0 .../storage/file/restoptions.go | 0 .../storage/file/util.go | 0 .../storage/file/watchset.go | 0 .../utils/clientConfig.go | 0 .../utils/meta.go | 0 .../utils/meta_test.go | 2 +- .../utils/tableConverter.go | 0 .../utils/tableConverter_test.go | 2 +- .../utils/uids.go | 0 .../wireset.go | 6 +- pkg/services/grafana-apiserver/config.go | 60 -- pkg/services/grafana-apiserver/config_test.go | 40 -- pkg/tests/apis/helper.go | 2 +- 93 files changed, 1102 insertions(+), 1446 deletions(-) delete mode 100644 pkg/aggregator/aggregator.go delete mode 100644 pkg/aggregator/config.go delete mode 100644 pkg/aggregator/crdRegistrationController.go delete mode 100644 pkg/cmd/grafana/apiserver/aggregator.md create mode 100644 pkg/registry/apis/datasource/querier.go rename pkg/services/{grafana-apiserver => apiserver}/README.md (100%) rename pkg/{ => services/apiserver}/aggregator/README.md (100%) create mode 100644 pkg/services/apiserver/aggregator/aggregator.go rename pkg/{ => services/apiserver}/aggregator/availableController.go (100%) rename pkg/{ => services/apiserver}/aggregator/resolver.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/impersonation.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/impersonation_test.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/org_id.go (95%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/org_role.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/provider.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/auth/authorizer/stack_id.go (94%) rename pkg/services/{grafana-apiserver => apiserver/builder}/common.go (94%) rename pkg/services/{grafana-apiserver => apiserver/builder}/helper.go (65%) rename pkg/services/{grafana-apiserver => apiserver/builder}/openapi.go (99%) rename pkg/services/{grafana-apiserver => apiserver/builder}/request_handler.go (96%) create mode 100644 pkg/services/apiserver/config.go rename pkg/services/{grafana-apiserver => apiserver}/endpoints/request/namespace.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/endpoints/request/namespace_test.go (97%) create mode 100644 pkg/services/apiserver/options/aggregator.go create mode 100644 pkg/services/apiserver/options/extra.go rename pkg/services/{grafana-apiserver => apiserver/options}/log.go (97%) create mode 100644 pkg/services/apiserver/options/options.go create mode 100644 pkg/services/apiserver/options/storage.go rename pkg/services/{grafana-apiserver => apiserver}/registry/generic/strategy.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/rest/dualwriter.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/service.go (63%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/fieldRequirements.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/restoptions.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/storage.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/utils.go (98%) rename pkg/services/{grafana-apiserver => apiserver}/storage/entity/utils_test.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/file.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/restoptions.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/util.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/storage/file/watchset.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/clientConfig.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/meta.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/meta_test.go (98%) rename pkg/services/{grafana-apiserver => apiserver}/utils/tableConverter.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/utils/tableConverter_test.go (98%) rename pkg/services/{grafana-apiserver => apiserver}/utils/uids.go (100%) rename pkg/services/{grafana-apiserver => apiserver}/wireset.go (63%) delete mode 100644 pkg/services/grafana-apiserver/config.go delete mode 100644 pkg/services/grafana-apiserver/config_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1ccee149cb5..6bd4a84f4e4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -108,7 +108,7 @@ /pkg/services/dashboardversion/ @grafana/backend-platform /pkg/services/encryption/ @grafana/backend-platform /pkg/services/folder/ @grafana/backend-platform -/pkg/services/grafana-apiserver @grafana/grafana-app-platform-squad +/pkg/services/apiserver @grafana/grafana-app-platform-squad /pkg/services/hooks/ @grafana/backend-platform /pkg/services/kmsproviders/ @grafana/backend-platform /pkg/services/licensing/ @grafana/backend-platform @@ -275,7 +275,6 @@ /pkg/modules/ @grafana/grafana-app-platform-squad /pkg/kindsysreport/ @grafana/grafana-app-platform-squad /pkg/services/grpcserver/ @grafana/grafana-app-platform-squad -/pkg/aggregator @grafana/grafana-app-platform-squad /pkg/generated @grafana/grafana-app-platform-squad # Alerting diff --git a/pkg/aggregator/aggregator.go b/pkg/aggregator/aggregator.go deleted file mode 100644 index 74bc2118847..00000000000 --- a/pkg/aggregator/aggregator.go +++ /dev/null @@ -1,628 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/aggregator.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/apiserver/apiextensions.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. - -package aggregator - -import ( - "crypto/tls" - "fmt" - "io" - "net" - "net/http" - "path" - "strings" - "sync" - "time" - - servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1" - serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned" - informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions" - "github.com/grafana/grafana/pkg/registry/apis/service" - grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver" - filestorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/file" - - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" - apiextensionsopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" - openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" - genericfeatures "k8s.io/apiserver/pkg/features" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/server/resourceconfig" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/apiserver/pkg/util/openapi" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" - apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - apiregistrationclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" - apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" - apiregistrationInformers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" - "k8s.io/kube-aggregator/pkg/controllers/autoregister" - apiserver "k8s.io/kube-aggregator/pkg/controllers/status" - aggregatoropenapi "k8s.io/kube-aggregator/pkg/generated/openapi" - "k8s.io/kube-openapi/pkg/common" -) - -// AggregatorServerOptions contains the state for the aggregator apiserver -type AggregatorServerOptions struct { - Builders []grafanaAPIServer.APIGroupBuilder - AlternateDNS []string - Config *Config - serviceResolver ServiceResolver - - sharedInformerFactory informersv0alpha1.SharedInformerFactory - - StdOut io.Writer - StdErr io.Writer -} - -func NewAggregatorServerOptions(out, errOut io.Writer, - options *options.RecommendedOptions, - extraConfig *ExtraConfig, -) (*AggregatorServerOptions, error) { - sharedConfig, err := initSharedConfig(options, aggregatorscheme.Codecs, nil) - if err != nil { - klog.Errorf("Error creating shared config: %s", err) - return nil, err - } - - sharedInformerFactory, err := initSharedInformerFactory(sharedConfig) - if err != nil { - klog.Errorf("Error creating shared informer factory: %s", err) - return nil, err - } - - serviceResolver, err := initServiceResolver(sharedInformerFactory) - if err != nil { - klog.Errorf("Error creating service resolver: %s", err) - return nil, err - } - - fakeInformers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute) - builders := []grafanaAPIServer.APIGroupBuilder{ - service.NewServiceAPIBuilder(), - } - - extensionsConfig, err := initApiExtensionsConfig(options, sharedConfig, fakeInformers, serviceResolver, extraConfig.DataPath) - if err != nil { - klog.Errorf("Error creating extensions config: %s", err) - return nil, err - } - - aggregatorConfig, err := initAggregatorConfig(options, sharedConfig, extraConfig, fakeInformers, builders, serviceResolver, extraConfig.DataPath) - if err != nil { - klog.Errorf("Error creating aggregator config: %s", err) - return nil, err - } - - return &AggregatorServerOptions{ - StdOut: out, - StdErr: errOut, - Builders: builders, - sharedInformerFactory: sharedInformerFactory, - serviceResolver: serviceResolver, - Config: &Config{ - Aggregator: aggregatorConfig, - ApiExtensions: extensionsConfig, - - SharedConfig: sharedConfig, - extraConfig: extraConfig, - }, - }, nil -} - -func (o *AggregatorServerOptions) LoadAPIGroupBuilders() error { - // Install schemas - for _, b := range o.Builders { - if err := b.InstallSchema(aggregatorscheme.Scheme); err != nil { - return err - } - } - return nil -} - -func initSharedConfig(options *options.RecommendedOptions, codecs serializer.CodecFactory, alternateDNS []string) (*genericapiserver.RecommendedConfig, error) { - if err := options.SecureServing.MaybeDefaultWithSelfSignedCerts( - "localhost", alternateDNS, []net.IP{net.IPv4(127, 0, 0, 1)}, - ); err != nil { - return nil, fmt.Errorf("error creating self-signed certificates: %v", err) - } - - options.Authentication.RemoteKubeConfigFileOptional = true - options.Authorization.RemoteKubeConfigFileOptional = true - - options.Admission = nil - - if options.CoreAPI.CoreAPIKubeconfigPath == "" { - options.CoreAPI = nil - } - - serverConfig := genericapiserver.NewRecommendedConfig(codecs) - - // NOTE: AggregatedDiscoveryGroupManager in kube-apiserver is set up by controlplane APIServerConfig creation - // Here, we adopt that one line in addition to what recommendedOptions gives us - // Without it, CRDs work on API routes (and are registered in openapi) but not discoverable by kubectl - serverConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis") - - if options.CoreAPI == nil { - if err := modifiedApplyTo(options, serverConfig); err != nil { - return nil, err - } - } else { - if err := options.ApplyTo(serverConfig); err != nil { - return nil, err - } - } - - return serverConfig, nil -} - -// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing -// a panic when CoreAPI is set to nil -func modifiedApplyTo(options *options.RecommendedOptions, config *genericapiserver.RecommendedConfig) error { - if err := options.Etcd.ApplyTo(&config.Config); err != nil { - return err - } - if err := options.EgressSelector.ApplyTo(&config.Config); err != nil { - return err - } - if err := options.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil { - return err - } - if err := options.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil { - return err - } - if err := options.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil { - return err - } - if err := options.Authorization.ApplyTo(&config.Config.Authorization); err != nil { - return err - } - if err := options.Audit.ApplyTo(&config.Config); err != nil { - return err - } - - // TODO: determine whether we need flow control (API priority and fairness) - //if err := options.Features.ApplyTo(&config.Config); err != nil { - // return err - //} - - if err := options.CoreAPI.ApplyTo(config); err != nil { - return err - } - - _, err := options.ExtraAdmissionInitializers(config) - if err != nil { - return err - } - return nil -} - -func getMergedOpenAPIDefinitions(builders []grafanaAPIServer.APIGroupBuilder, ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { - // Add OpenAPI specs for each group+version - prerequisiteAPIs := grafanaAPIServer.GetOpenAPIDefinitions(builders)(ref) - aggregatorAPIs := aggregatoropenapi.GetOpenAPIDefinitions(ref) - - for k, v := range prerequisiteAPIs { - aggregatorAPIs[k] = v - } - - return aggregatorAPIs -} - -func initSharedInformerFactory(sharedConfig *genericapiserver.RecommendedConfig) (informersv0alpha1.SharedInformerFactory, error) { - serviceClient, err := serviceclientset.NewForConfig(sharedConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - return informersv0alpha1.NewSharedInformerFactory( - serviceClient, - 5*time.Minute, // this is effectively used as a refresh interval right now. Might want to do something nicer later on. - ), nil -} - -func initServiceResolver(factory informersv0alpha1.SharedInformerFactory) (apiserver.ServiceResolver, error) { - return NewExternalNameResolver(factory.Service().V0alpha1().ExternalNames().Lister()), nil -} - -func initApiExtensionsConfig(options *options.RecommendedOptions, - sharedConfig *genericapiserver.RecommendedConfig, - fakeInfomers informers.SharedInformerFactory, - serviceResolver apiserver.ServiceResolver, - dataPath string, -) (*apiextensionsapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the api extensions - genericConfig := sharedConfig.Config - - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - - // copy the etcd options so we don't mutate originals. - // we assume that the etcd options have been completed already. avoid messing with anything outside - // of changes to StorageConfig as that may lead to unexpected behavior when the options are applied. - etcdOptions := *options.Etcd - // this is where the true decodable levels come from. - etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) - // prefer the more compact serialization (v1beta1) for storage until https://issue.k8s.io/82292 is resolved for objects whose v1 serialization is too big but whose v1beta1 serialization can be stored - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(apiextensionsv1beta1.SchemeGroupVersion, schema.GroupKind{Group: apiextensionsv1beta1.GroupName}) - etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks - if err := etcdOptions.ApplyTo(&genericConfig); err != nil { - return nil, err - } - - restOptionsGetter := filestorage.NewRESTOptionsGetter(path.Join(dataPath, "grafana-apiextensionsserver"), etcdOptions.StorageConfig) - genericConfig.RESTOptionsGetter = restOptionsGetter - - // NOTE: ignoring genericConfig.ResourceTransformers in crdOptionsGetter creation for now - // crdOptionsGetter := apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions, genericConfig.ResourceTransformers, ) - // The following is equivalent code to apiextensionsoptions.NewCRDRESTOptionsGetter with lesser dependencies - crdEtcdOptions := etcdOptions - crdEtcdOptions.StorageConfig.Codec = unstructured.UnstructuredJSONScheme - crdEtcdOptions.StorageConfig.StorageObjectCountTracker = genericConfig.StorageObjectCountTracker - crdEtcdOptions.WatchCacheSizes = nil // this control is not provided for custom resources - - // override MergedResourceConfig with apiextensions defaults and registry - mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(apiextensionsapiserver.DefaultAPIResourceConfigSource(), nil, apiextensionsapiserver.Scheme) - if err != nil { - return nil, err - } - genericConfig.MergedResourceConfig = mergedResourceConfig - - genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(apiextensionsopenapi.GetOpenAPIDefinitions), openapinamer.NewDefinitionNamer(apiextensionsapiserver.Scheme, apiextensionsapiserver.Scheme)) - - apiextensionsConfig := &apiextensionsapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: fakeInfomers, - }, - ExtraConfig: apiextensionsapiserver.ExtraConfig{ - CRDRESTOptionsGetter: filestorage.NewRESTOptionsGetter(path.Join(dataPath, "grafana-apiextensionsserver"), crdEtcdOptions.StorageConfig), - // TODO: remove the hardcod when HA story is more developed - MasterCount: 1, - // TODO: leaving AuthResolverWrapper unset doesn't impact basic operation of CRDs - // AuthResolverWrapper: authResolverWrapper, - ServiceResolver: serviceResolver, - }, - } - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - apiextensionsConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return apiextensionsConfig, nil -} - -func initAggregatorConfig(options *options.RecommendedOptions, - sharedConfig *genericapiserver.RecommendedConfig, - extra *ExtraConfig, - fakeInformers informers.SharedInformerFactory, - builders []grafanaAPIServer.APIGroupBuilder, - serviceResolver apiserver.ServiceResolver, - dataPath string, -) (*aggregatorapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator - genericConfig := sharedConfig.Config - - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - // prevent generic API server from installing the OpenAPI handler. Aggregator server - // has its own customized OpenAPI handler. - genericConfig.SkipOpenAPIInstallation = true - mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(aggregatorapiserver.DefaultAPIResourceConfigSource(), nil, aggregatorscheme.Scheme) - if err != nil { - return nil, err - } - genericConfig.MergedResourceConfig = mergedResourceConfig - - getOpenAPIDefinitionsFunc := func() common.GetOpenAPIDefinitions { - return func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { - return getMergedOpenAPIDefinitions(builders, ref) - } - } - - namer := openapinamer.NewDefinitionNamer(aggregatorscheme.Scheme, apiextensionsapiserver.Scheme) - genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitionsFunc(), namer) - genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes" - genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitionsFunc(), namer) - genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" - - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) && - utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { - // Add StorageVersionPrecondition handler to aggregator-apiserver. - // The handler will block write requests to built-in resources until the - // target resources' storage versions are up-to-date. - genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition - } - - // copy the etcd options so we don't mutate originals. - // we assume that the etcd options have been completed already. avoid messing with anything outside - // of changes to StorageConfig as that may lead to unexpected behavior when the options are applied. - etcdOptions := *options.Etcd - etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion, - apiregistrationv1beta1.SchemeGroupVersion, - servicev0alpha1.SchemeGroupVersion) - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion, - schema.GroupKind{Group: apiregistrationv1beta1.GroupName}, - schema.GroupKind{Group: servicev0alpha1.GROUP}) - etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks - if err := etcdOptions.ApplyTo(&genericConfig); err != nil { - return nil, err - } - genericConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(path.Join(dataPath, "grafana-aggregator"), etcdOptions.StorageConfig) - - genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("apiservice-status-available-controller") - genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("start-kube-aggregator-informers") - - aggregatorConfig := &aggregatorapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: fakeInformers, - ClientConfig: genericConfig.LoopbackClientConfig, - }, - ExtraConfig: aggregatorapiserver.ExtraConfig{ - ProxyClientCertFile: extra.ProxyClientCertFile, - ProxyClientKeyFile: extra.ProxyClientKeyFile, - // NOTE: while ProxyTransport can be skipped in the configuration, it allows honoring - // DISABLE_HTTP2, HTTPS_PROXY and NO_PROXY env vars as needed - ProxyTransport: createProxyTransport(), - }, - } - - aggregatorConfig.ExtraConfig.ServiceResolver = serviceResolver - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - aggregatorConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return aggregatorConfig, nil -} - -func (o *AggregatorServerOptions) CreateAggregatorServer(delegateAPIServer genericapiserver.DelegationTarget, apiExtensionsInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) { - completedConfig := o.Config.AggregatorComplete - aggregatorServer, err := completedConfig.NewWithDelegate(delegateAPIServer) - if err != nil { - return nil, err - } - - // create controllers for auto-registration - apiRegistrationClient, err := apiregistrationclient.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient) - apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController) - - crdRegistrationController := NewCRDRegistrationController( - apiExtensionsInformers.Apiextensions().V1().CustomResourceDefinitions(), - autoRegistrationController) - - // Imbue all builtin group-priorities onto the aggregated discovery - if completedConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil { - for gv, entry := range apiVersionPriorities { - completedConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.group), int(entry.version)) - } - } - - err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error { - go crdRegistrationController.Run(5, context.StopCh) - go func() { - crdRegistrationController.WaitForInitialSync() - autoRegistrationController.Run(5, context.StopCh) - }() - return nil - }) - if err != nil { - return nil, err - } - - err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks( - makeAPIServiceAvailableHealthCheck( - "autoregister-completion", - apiServices, - aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), - ), - ) - if err != nil { - return nil, err - } - - apiregistrationClient, err := apiregistrationclientset.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - availableController, err := NewAvailableConditionController( - aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), - o.sharedInformerFactory.Service().V0alpha1().ExternalNames(), - apiregistrationClient.ApiregistrationV1(), - nil, - (func() ([]byte, []byte))(nil), - completedConfig.ExtraConfig.ServiceResolver, - ) - if err != nil { - return nil, err - } - - aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-override-available-controller", func(context genericapiserver.PostStartHookContext) error { - // if we end up blocking for long periods of time, we may need to increase workers. - go availableController.Run(5, context.StopCh) - return nil - }) - - aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("start-grafana-aggregator-informers", func(context genericapiserver.PostStartHookContext) error { - o.sharedInformerFactory.Start(context.StopCh) - aggregatorServer.APIRegistrationInformers.Start(context.StopCh) - return nil - }) - - // Install the API Group+version - for _, b := range o.Builders { - g, err := b.GetAPIGroupInfo(aggregatorscheme.Scheme, aggregatorscheme.Codecs, completedConfig.GenericConfig.RESTOptionsGetter) - if err != nil { - return nil, err - } - if g == nil || len(g.PrioritizedVersions) < 1 { - continue - } - err = aggregatorServer.GenericAPIServer.InstallAPIGroup(g) - if err != nil { - return nil, err - } - } - - return aggregatorServer, nil -} - -func makeAPIService(gv schema.GroupVersion) *v1.APIService { - apiServicePriority, ok := apiVersionPriorities[gv] - if !ok { - // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version - // being permanently stuck in the APIServices list. - klog.Infof("Skipping APIService creation for %v", gv) - return nil - } - return &v1.APIService{ - ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, - Spec: v1.APIServiceSpec{ - Group: gv.Group, - Version: gv.Version, - GroupPriorityMinimum: apiServicePriority.group, - VersionPriority: apiServicePriority.version, - }, - } -} - -// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy -// once all of the specified services have been observed to be available at least once. -func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer apiregistrationInformers.APIServiceInformer) healthz.HealthChecker { - // Track the auto-registered API services that have not been observed to be available yet - pendingServiceNamesLock := &sync.RWMutex{} - pendingServiceNames := sets.NewString() - for _, service := range apiServices { - pendingServiceNames.Insert(service.Name) - } - - // When an APIService in the list is seen as available, remove it from the pending list - handleAPIServiceChange := func(service *v1.APIService) { - pendingServiceNamesLock.Lock() - defer pendingServiceNamesLock.Unlock() - if !pendingServiceNames.Has(service.Name) { - return - } - if v1helper.IsAPIServiceConditionTrue(service, v1.Available) { - pendingServiceNames.Delete(service.Name) - } - } - - // Watch add/update events for APIServices - _, _ = apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) }, - UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) }, - }) - - // Don't return healthy until the pending list is empty - return healthz.NamedCheck(name, func(r *http.Request) error { - pendingServiceNamesLock.RLock() - defer pendingServiceNamesLock.RUnlock() - if pendingServiceNames.Len() > 0 { - return fmt.Errorf("missing APIService: %v", pendingServiceNames.List()) - } - return nil - }) -} - -// priority defines group priority that is used in discovery. This controls -// group position in the kubectl output. -type priority struct { - // group indicates the order of the group relative to other groups. - group int32 - // version indicates the relative order of the version inside of its group. - version int32 -} - -// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers -// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. -// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. -// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. -var apiVersionPriorities = map[schema.GroupVersion]priority{ - {Group: "", Version: "v1"}: {group: 18000, version: 1}, - // to my knowledge, nothing below here collides - {Group: "admissionregistration.k8s.io", Version: "v1"}: {group: 16700, version: 15}, - {Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {group: 16700, version: 12}, - {Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {group: 16700, version: 9}, - {Group: "apiextensions.k8s.io", Version: "v1"}: {group: 16700, version: 15}, - // Append a new group to the end of the list if unsure. - // You can use min(existing group)-100 as the initial value for a group. - // Version can be set to 9 (to have space around) for a new group. -} - -func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService { - apiServices := []*v1.APIService{} - - for _, curr := range delegateAPIServer.ListedPaths() { - if curr == "/api/v1" { - apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"}) - registration.AddAPIServiceToSyncOnStart(apiService) - apiServices = append(apiServices, apiService) - continue - } - - if !strings.HasPrefix(curr, "/apis/") { - continue - } - // this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1 - tokens := strings.Split(curr, "/") - if len(tokens) != 4 { - continue - } - - apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) - if apiService == nil { - continue - } - registration.AddAPIServiceToSyncOnStart(apiService) - apiServices = append(apiServices, apiService) - } - - return apiServices -} - -// NOTE: below function imported from https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L197 -// createProxyTransport creates the dialer infrastructure to connect to the api servers. -func createProxyTransport() *http.Transport { - // NOTE: We don't set proxyDialerFn but the below SetTransportDefaults will - // See https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/net/http.go#L109 - var proxyDialerFn utilnet.DialFunc - // Proxying to services is IP-based... don't expect to be able to verify the hostname - proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} - proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ - DialContext: proxyDialerFn, - TLSClientConfig: proxyTLSClientConfig, - }) - return proxyTransport -} diff --git a/pkg/aggregator/config.go b/pkg/aggregator/config.go deleted file mode 100644 index e7bcd84e286..00000000000 --- a/pkg/aggregator/config.go +++ /dev/null @@ -1,57 +0,0 @@ -package aggregator - -import ( - "github.com/spf13/pflag" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" -) - -type ExtraConfig struct { - ProxyClientCertFile string - ProxyClientKeyFile string - - DataPath string -} - -type Config struct { - Aggregator *aggregatorapiserver.Config - ApiExtensions *apiextensionsapiserver.Config - - AggregatorComplete aggregatorapiserver.CompletedConfig - ApiExtensionsComplete apiextensionsapiserver.CompletedConfig - - recommendedOptions *options.RecommendedOptions - SharedConfig *genericapiserver.RecommendedConfig - extraConfig *ExtraConfig -} - -func (c *Config) AddFlags(fs *pflag.FlagSet) { - if c == nil { - return - } - - c.recommendedOptions.AddFlags(fs) -} - -func (c *Config) Complete() { - if c == nil { - return - } - - c.ApiExtensionsComplete = c.ApiExtensions.Complete() - c.AggregatorComplete = c.Aggregator.Complete() -} - -func (ec *ExtraConfig) AddFlags(fs *pflag.FlagSet) { - if ec == nil { - return - } - - fs.StringVar(&ec.ProxyClientCertFile, "proxy-client-cert-file", ec.ProxyClientCertFile, - "path to proxy client cert file") - - fs.StringVar(&ec.ProxyClientKeyFile, "proxy-client-key-file", ec.ProxyClientKeyFile, - "path to proxy client cert file") -} diff --git a/pkg/aggregator/crdRegistrationController.go b/pkg/aggregator/crdRegistrationController.go deleted file mode 100644 index f3bbdd3c85d..00000000000 --- a/pkg/aggregator/crdRegistrationController.go +++ /dev/null @@ -1,214 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/controller/crdregistration/crdregistration_controller.go -// Provenance-includes-license: Apache-2.0 -// Provenance-includes-copyright: The Kubernetes Authors. - -package aggregator - -import ( - "fmt" - "time" - - "k8s.io/klog/v2" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - crdinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1" - crdlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" -) - -// AutoAPIServiceRegistration is an interface which callers can re-declare locally and properly cast to for -// adding and removing APIServices -type AutoAPIServiceRegistration interface { - // AddAPIServiceToSync adds an API service to auto-register. - AddAPIServiceToSync(in *v1.APIService) - // RemoveAPIServiceToSync removes an API service to auto-register. - RemoveAPIServiceToSync(name string) -} - -type crdRegistrationController struct { - crdLister crdlisters.CustomResourceDefinitionLister - crdSynced cache.InformerSynced - - apiServiceRegistration AutoAPIServiceRegistration - - syncHandler func(groupVersion schema.GroupVersion) error - - syncedInitialSet chan struct{} - - // queue is where incoming work is placed to de-dup and to allow "easy" rate limited requeues on errors - // this is actually keyed by a groupVersion - queue workqueue.RateLimitingInterface -} - -// NewCRDRegistrationController returns a controller which will register CRD GroupVersions with the auto APIService registration -// controller so they automatically stay in sync. -func NewCRDRegistrationController(crdinformer crdinformers.CustomResourceDefinitionInformer, apiServiceRegistration AutoAPIServiceRegistration) *crdRegistrationController { - c := &crdRegistrationController{ - crdLister: crdinformer.Lister(), - crdSynced: crdinformer.Informer().HasSynced, - apiServiceRegistration: apiServiceRegistration, - syncedInitialSet: make(chan struct{}), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_autoregistration_controller"), - } - c.syncHandler = c.handleVersionUpdate - - _, _ = crdinformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - cast := obj.(*apiextensionsv1.CustomResourceDefinition) - c.enqueueCRD(cast) - }, - UpdateFunc: func(oldObj, newObj interface{}) { - // Enqueue both old and new object to make sure we remove and add appropriate API services. - // The working queue will resolve any duplicates and only changes will stay in the queue. - c.enqueueCRD(oldObj.(*apiextensionsv1.CustomResourceDefinition)) - c.enqueueCRD(newObj.(*apiextensionsv1.CustomResourceDefinition)) - }, - DeleteFunc: func(obj interface{}) { - cast, ok := obj.(*apiextensionsv1.CustomResourceDefinition) - if !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - klog.V(2).Infof("Couldn't get object from tombstone %#v", obj) - return - } - cast, ok = tombstone.Obj.(*apiextensionsv1.CustomResourceDefinition) - if !ok { - klog.V(2).Infof("Tombstone contained unexpected object: %#v", obj) - return - } - } - c.enqueueCRD(cast) - }, - }) - - return c -} - -func (c *crdRegistrationController) Run(workers int, stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() - // make sure the work queue is shutdown which will trigger workers to end - defer c.queue.ShutDown() - - klog.Infof("Starting crd-autoregister controller") - defer klog.Infof("Shutting down crd-autoregister controller") - - // wait for your secondary caches to fill before starting your work - if !cache.WaitForNamedCacheSync("crd-autoregister", stopCh, c.crdSynced) { - return - } - - // process each item in the list once - if crds, err := c.crdLister.List(labels.Everything()); err != nil { - utilruntime.HandleError(err) - } else { - for _, crd := range crds { - for _, version := range crd.Spec.Versions { - if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}); err != nil { - utilruntime.HandleError(err) - } - } - } - } - close(c.syncedInitialSet) - - // start up your worker threads based on workers. Some controllers have multiple kinds of workers - for i := 0; i < workers; i++ { - // runWorker will loop until "something bad" happens. The .Until will then rekick the worker - // after one second - go wait.Until(c.runWorker, time.Second, stopCh) - } - - // wait until we're told to stop - <-stopCh -} - -// WaitForInitialSync blocks until the initial set of CRD resources has been processed -func (c *crdRegistrationController) WaitForInitialSync() { - <-c.syncedInitialSet -} - -func (c *crdRegistrationController) runWorker() { - // hot loop until we're told to stop. processNextWorkItem will automatically wait until there's work - // available, so we don't worry about secondary waits - for c.processNextWorkItem() { - } -} - -// processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. -func (c *crdRegistrationController) processNextWorkItem() bool { - // pull the next work item from queue. It should be a key we use to lookup something in a cache - key, quit := c.queue.Get() - if quit { - return false - } - // you always have to indicate to the queue that you've completed a piece of work - defer c.queue.Done(key) - - // do your work on the key. This method will contains your "do stuff" logic - err := c.syncHandler(key.(schema.GroupVersion)) - if err == nil { - // if you had no error, tell the queue to stop tracking history for your key. This will - // reset things like failure counts for per-item rate limiting - c.queue.Forget(key) - return true - } - - // there was a failure so be sure to report it. This method allows for pluggable error handling - // which can be used for things like cluster-monitoring - utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err)) - // since we failed, we should requeue the item to work on later. This method will add a backoff - // to avoid hotlooping on particular items (they're probably still not going to work right away) - // and overall controller protection (everything I've done is broken, this controller needs to - // calm down or it can starve other useful work) cases. - c.queue.AddRateLimited(key) - - return true -} - -func (c *crdRegistrationController) enqueueCRD(crd *apiextensionsv1.CustomResourceDefinition) { - for _, version := range crd.Spec.Versions { - c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}) - } -} - -func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.GroupVersion) error { - apiServiceName := groupVersion.Version + "." + groupVersion.Group - - // check all CRDs. There shouldn't that many, but if we have problems later we can index them - crds, err := c.crdLister.List(labels.Everything()) - if err != nil { - return err - } - for _, crd := range crds { - if crd.Spec.Group != groupVersion.Group { - continue - } - for _, version := range crd.Spec.Versions { - if version.Name != groupVersion.Version || !version.Served { - continue - } - - c.apiServiceRegistration.AddAPIServiceToSync(&v1.APIService{ - ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, - Spec: v1.APIServiceSpec{ - Group: groupVersion.Group, - Version: groupVersion.Version, - GroupPriorityMinimum: 1000, // CRDs should have relatively low priority - VersionPriority: 100, // CRDs will be sorted by kube-like versions like any other APIService with the same VersionPriority - }, - }) - return nil - } - } - - c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName) - return nil -} diff --git a/pkg/api/frontendsettings_test.go b/pkg/api/frontendsettings_test.go index 53a3a322377..faaf564c4da 100644 --- a/pkg/api/frontendsettings_test.go +++ b/pkg/api/frontendsettings_test.go @@ -19,8 +19,8 @@ import ( "github.com/grafana/grafana/pkg/plugins/config" "github.com/grafana/grafana/pkg/plugins/pluginscdn" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index a6895f7a6ce..c9a4536f2a1 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -25,8 +25,8 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/grafana/grafana/pkg/services/anonymous" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/api/avatar" "github.com/grafana/grafana/pkg/api/routing" diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index 4bbe662c081..df798af8ca8 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -13,10 +13,10 @@ import ( "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/middleware/requestmeta" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/util/errutil/errhttp" "github.com/grafana/grafana/pkg/web" ) diff --git a/pkg/api/playlist.go b/pkg/api/playlist.go index 4a49bc59708..9f8b2acb85f 100644 --- a/pkg/api/playlist.go +++ b/pkg/api/playlist.go @@ -15,10 +15,10 @@ import ( "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" "github.com/grafana/grafana/pkg/middleware" internalplaylist "github.com/grafana/grafana/pkg/registry/apis/playlist" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/util/errutil/errhttp" "github.com/grafana/grafana/pkg/web" diff --git a/pkg/cmd/grafana/apiserver/aggregator.md b/pkg/cmd/grafana/apiserver/aggregator.md deleted file mode 100644 index 0c8ae3fae9a..00000000000 --- a/pkg/cmd/grafana/apiserver/aggregator.md +++ /dev/null @@ -1,51 +0,0 @@ -# grafana aggregator - -The `aggregator` command in this binary is our equivalent of what kube-apiserver does for aggregation using -the `kube-aggregator` pkg. Here, we enable only select controllers that are useful for aggregation in a Grafana -cloud context. In future, Grafana microservices (and even plugins) will run as separate API servers -hosting each their own APIs (with specific Group/Versions). The `aggregator` component here shall act similar to what -`kube-apiserver` does: doing healthchecks for `APIService` objects registered against it and acting as a proxy for -the specified `GroupVersion` therein. - -## How to get started - -1. Generate the PKI using `openssl` (for development purposes, we will use the CN of `system:masters`): - ```shell - ./hack/make-aggregator-pki.sh - ``` -2. Start the aggregator: - ```shell - # This will generate the kubeconfig which you can use in the extension apiservers for - # enforcing delegate authnz under $PWD/data/grafana-apiserver/aggregator.kubeconfig - go run ./pkg/cmd/grafana aggregator --secure-port 8443 \ - --proxy-client-cert-file $PWD/data/grafana-aggregator/client.crt \ - --proxy-client-key-file $PWD/data/grafana-aggregator/client.key - ``` -3. Apply the manifests: - ```shell - export KUBECONFIG=$PWD/data/grafana-apiserver/aggregator.kubeconfig - kubectl apply -k ./pkg/cmd/grafana/apiserver/deploy/aggregator-test - # SAMPLE OUTPUT - # apiservice.apiregistration.k8s.io/v0alpha1.example.grafana.app created - # externalname.service.grafana.app/example-apiserver created - - kubectl get apiservice - # SAMPLE OUTPUT - # NAME SERVICE AVAILABLE AGE - # v0alpha1.example.grafana.app grafana/example-apiserver False (FailedDiscoveryCheck) 29m - ``` -4. In another tab, start the example microservice that will be aggregated by the parent apiserver: - ```shell - go run ./pkg/cmd/grafana apiserver example.grafana.app \ - --kubeconfig $PWD/data/grafana-aggregator/aggregator.kubeconfig \ - --secure-port 7443 \ - --client-ca-file=$PWD/data/grafana-aggregator/ca.crt - ``` -5. Check `APIService` again: - ```shell - export KUBECONFIG=$PWD/data/grafana-apiserver/aggregator.kubeconfig - kubectl get apiservice - # SAMPLE OUTPUT - # NAME SERVICE AVAILABLE AGE - # v0alpha1.example.grafana.app grafana/example-apiserver True 30m - ``` diff --git a/pkg/cmd/grafana/apiserver/cmd.go b/pkg/cmd/grafana/apiserver/cmd.go index 338c1ca9662..d24829dba19 100644 --- a/pkg/cmd/grafana/apiserver/cmd.go +++ b/pkg/cmd/grafana/apiserver/cmd.go @@ -2,26 +2,13 @@ package apiserver import ( "os" - "path" "github.com/spf13/cobra" - genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/util/notfoundhandler" - "k8s.io/client-go/tools/clientcmd" "k8s.io/component-base/cli" - "k8s.io/klog/v2" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - "github.com/grafana/grafana/pkg/aggregator" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" -) - -const ( - aggregatorDataPath = "data" - defaultAggregatorEtcdPathPrefix = "/registry/grafana.aggregator" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" ) func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}) *cobra.Command { @@ -74,97 +61,3 @@ func RunCLI() int { return cli.Run(cmd) } - -func newCommandStartAggregator() *cobra.Command { - devAcknowledgementNotice := "The aggregator command is in heavy development. The entire setup is subject to change without notice" - - cwd, err := os.Getwd() - if err != nil { - panic("could not determine current directory") - } - - extraConfig := &aggregator.ExtraConfig{ - DataPath: path.Join(cwd, aggregatorDataPath), - } - - // Register standard k8s flags with the command line - recommendedOptions := options.NewRecommendedOptions( - defaultAggregatorEtcdPathPrefix, - aggregatorscheme.Codecs.LegacyCodec(), // codec is passed to etcd and hence not used - ) - - cmd := &cobra.Command{ - Use: "aggregator", - Short: "Run the grafana aggregator", - Long: "Run a standalone kubernetes based aggregator server. " + - devAcknowledgementNotice, - Example: "grafana aggregator", - RunE: func(c *cobra.Command, args []string) error { - serverOptions, err := aggregator.NewAggregatorServerOptions(os.Stdout, os.Stderr, recommendedOptions, extraConfig) - serverOptions.Config.Complete() - - if err != nil { - klog.Errorf("Could not create aggregator server options: %s", err) - os.Exit(1) - } - - return run(serverOptions) - }, - } - - recommendedOptions.AddFlags(cmd.Flags()) - extraConfig.AddFlags(cmd.Flags()) - - return cmd -} - -func run(serverOptions *aggregator.AggregatorServerOptions) error { - if err := serverOptions.LoadAPIGroupBuilders(); err != nil { - klog.Errorf("Error loading prerequisite APIs: %s", err) - return err - } - - notFoundHandler := notfoundhandler.New(serverOptions.Config.SharedConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) - apiExtensionsServer, err := serverOptions.Config.ApiExtensionsComplete.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler)) - if err != nil { - return err - } - - aggregator, err := serverOptions.CreateAggregatorServer(apiExtensionsServer.GenericAPIServer, apiExtensionsServer.Informers) - if err != nil { - klog.Errorf("Error creating aggregator server: %s", err) - return err - } - - // Install the API Group+version - err = grafanaapiserver.InstallAPIs(aggregator.GenericAPIServer, serverOptions.Config.Aggregator.GenericConfig.RESTOptionsGetter, serverOptions.Builders) - if err != nil { - klog.Errorf("Error installing apis: %s", err) - return err - } - - if err := clientcmd.WriteToFile( - utils.FormatKubeConfig(aggregator.GenericAPIServer.LoopbackClientConfig), - path.Join(aggregatorDataPath, "grafana-aggregator", "aggregator.kubeconfig"), - ); err != nil { - klog.Errorf("Error persisting aggregator.kubeconfig: %s", err) - return err - } - - prepared, err := aggregator.PrepareRun() - if err != nil { - return err - } - - stopCh := genericapiserver.SetupSignalHandler() - if err := prepared.Run(stopCh); err != nil { - return err - } - return nil -} - -func RunCobraWrapper() int { - cmd := newCommandStartAggregator() - - return cli.Run(cmd) -} diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index d40f89d615b..a8f5fffb3e7 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -17,9 +17,10 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/query" "github.com/grafana/grafana/pkg/registry/apis/query/runner" "github.com/grafana/grafana/pkg/server" + grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/setting" ) @@ -30,7 +31,7 @@ const ( // APIServerOptions contains the state for the apiserver type APIServerOptions struct { - builders []grafanaAPIServer.APIGroupBuilder + builders []builder.APIGroupBuilder RecommendedOptions *options.RecommendedOptions AlternateDNS []string @@ -46,7 +47,7 @@ func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions { } func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error { - o.builders = []grafanaAPIServer.APIGroupBuilder{} + o.builders = []builder.APIGroupBuilder{} for _, g := range args { switch g { // No dependencies for testing @@ -171,7 +172,7 @@ func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) serverConfig.DisabledPostStartHooks = serverConfig.DisabledPostStartHooks.Insert("priority-and-fairness-config-consumer") // Add OpenAPI specs for each group+version - err := grafanaAPIServer.SetupConfig(serverConfig, o.builders) + err := builder.SetupConfig(grafanaAPIServer.Scheme, serverConfig, o.builders) return serverConfig, err } @@ -199,7 +200,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf } // Install the API Group+version - err = grafanaAPIServer.InstallAPIs(server, config.RESTOptionsGetter, o.builders) + err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, true) if err != nil { return err } diff --git a/pkg/cmd/grafana/main.go b/pkg/cmd/grafana/main.go index d86e66c3a21..696198e40b9 100644 --- a/pkg/cmd/grafana/main.go +++ b/pkg/cmd/grafana/main.go @@ -46,18 +46,6 @@ func main() { }, }, gsrv.ServerCommand(version, commit, enterpriseCommit, buildBranch, buildstamp), - { - // The kube-aggregator inspired grafana aggregator - Name: "aggregator", - Usage: "run grafana aggregator (experimental)", - // Skip parsing flags because the command line is actually managed by cobra - SkipFlagParsing: true, - Action: func(context *cli.Context) error { - // exit here because apiserver handles its own error output - os.Exit(apiserver.RunCobraWrapper()) - return nil - }, - }, }, CommandNotFound: cmdNotFound, EnableBashCompletion: true, diff --git a/pkg/registry/apis/apis.go b/pkg/registry/apis/apis.go index f83ac018bda..86f757eb8ba 100644 --- a/pkg/registry/apis/apis.go +++ b/pkg/registry/apis/apis.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/folders" "github.com/grafana/grafana/pkg/registry/apis/playlist" "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/service" ) var ( @@ -28,6 +29,7 @@ func ProvideRegistryServiceSink( _ *featuretoggle.FeatureFlagAPIBuilder, _ *datasource.DataSourceAPIBuilder, _ *folders.FolderAPIBuilder, + _ *service.ServiceAPIBuilder, _ *query.QueryAPIBuilder, ) *Service { return &Service{} diff --git a/pkg/registry/apis/dashboard/access/sql_dashboards.go b/pkg/registry/apis/dashboard/access/sql_dashboards.go index b0e8304fa6f..4064a1fa7a3 100644 --- a/pkg/registry/apis/dashboard/access/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/access/sql_dashboards.go @@ -14,9 +14,9 @@ import ( "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/sqlstore/session" ) diff --git a/pkg/registry/apis/dashboard/authorizer.go b/pkg/registry/apis/dashboard/authorizer.go index 9aa8d2b0f59..54a772591d5 100644 --- a/pkg/registry/apis/dashboard/authorizer.go +++ b/pkg/registry/apis/dashboard/authorizer.go @@ -7,8 +7,8 @@ import ( "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/guardian" ) diff --git a/pkg/registry/apis/dashboard/legacy_storage.go b/pkg/registry/apis/dashboard/legacy_storage.go index 73062163836..e5a3a5ede65 100644 --- a/pkg/registry/apis/dashboard/legacy_storage.go +++ b/pkg/registry/apis/dashboard/legacy_storage.go @@ -14,7 +14,7 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/dashboard/access" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" ) var ( diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index 7acf642e0d1..e6434240b15 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -19,19 +19,19 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/registry/apis/dashboard/access" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" dashver "github.com/grafana/grafana/pkg/services/dashboardversion" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/setting" ) -var _ grafanaapiserver.APIGroupBuilder = (*DashboardsAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*DashboardsAPIBuilder)(nil) // This is used just so wire has something unique to return type DashboardsAPIBuilder struct { @@ -47,7 +47,7 @@ type DashboardsAPIBuilder struct { } func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, dashboardService dashboards.DashboardService, dashboardVersionService dashver.Service, accessControl accesscontrol.AccessControl, @@ -113,6 +113,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -171,7 +172,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( } // Dual writes if a RESTOptionsGetter is provided - if optsGetter != nil { + if dualWrite && optsGetter != nil { options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: grafanaregistry.GetAttrs} if err := store.CompleteWithOptions(options); err != nil { return nil, err @@ -195,6 +196,6 @@ func (b *DashboardsAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefiniti return v0alpha1.GetOpenAPIDefinitions } -func (b *DashboardsAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *DashboardsAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil // no custom API routes } diff --git a/pkg/registry/apis/dashboard/sub_access.go b/pkg/registry/apis/dashboard/sub_access.go index c184cc8adce..ca2c783b5e2 100644 --- a/pkg/registry/apis/dashboard/sub_access.go +++ b/pkg/registry/apis/dashboard/sub_access.go @@ -11,9 +11,9 @@ import ( "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/auth/identity" dashboardssvc "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/guardian" ) diff --git a/pkg/registry/apis/dashboard/sub_versions.go b/pkg/registry/apis/dashboard/sub_versions.go index ca4c06caba1..a38f9947d84 100644 --- a/pkg/registry/apis/dashboard/sub_versions.go +++ b/pkg/registry/apis/dashboard/sub_versions.go @@ -13,8 +13,8 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" dashver "github.com/grafana/grafana/pkg/services/dashboardversion" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) type VersionsREST struct { diff --git a/pkg/registry/apis/dashboard/summary_storage.go b/pkg/registry/apis/dashboard/summary_storage.go index ef9cf3578ec..17311be1372 100644 --- a/pkg/registry/apis/dashboard/summary_storage.go +++ b/pkg/registry/apis/dashboard/summary_storage.go @@ -10,7 +10,7 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/dashboard/access" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" ) var ( diff --git a/pkg/registry/apis/datasource/plugincontext.go b/pkg/registry/apis/datasource/plugincontext.go index d4c9e01f5b3..1c39aaa9e26 100644 --- a/pkg/registry/apis/datasource/plugincontext.go +++ b/pkg/registry/apis/datasource/plugincontext.go @@ -9,9 +9,9 @@ import ( "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" ) diff --git a/pkg/registry/apis/datasource/querier.go b/pkg/registry/apis/datasource/querier.go new file mode 100644 index 00000000000..9d5b9ff4702 --- /dev/null +++ b/pkg/registry/apis/datasource/querier.go @@ -0,0 +1,148 @@ +package datasource + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" + "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/datasources" +) + +type QuerierFactoryFunc func(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) + +type QuerierProvider interface { + Querier(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) +} + +type DefaultQuerierProvider struct { + factory QuerierFactoryFunc +} + +func ProvideDefaultQuerierProvider(pluginClient plugins.Client, dsService datasources.DataSourceService, + dsCache datasources.CacheService) *DefaultQuerierProvider { + return NewQuerierProvider(func(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) { + return NewDefaultQuerier(ri, pj, pluginClient, dsService, dsCache), nil + }) +} + +func NewQuerierProvider(factory QuerierFactoryFunc) *DefaultQuerierProvider { + return &DefaultQuerierProvider{ + factory: factory, + } +} + +func (p *DefaultQuerierProvider) Querier(ctx context.Context, ri common.ResourceInfo, pj plugins.JSONData) (Querier, error) { + return p.factory(ctx, ri, pj) +} + +// Querier is the interface that wraps the Query method. +type Querier interface { + // Query runs the query on behalf of the user in context. + Query(ctx context.Context, query *backend.QueryDataRequest) (*backend.QueryDataResponse, error) + // Health checks the health of the plugin. + Health(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) + // Resource gets a resource plugin. + Resource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error + // Datasource gets all data source plugins (with elevated permissions). + Datasource(ctx context.Context, name string) (*v0alpha1.DataSourceConnection, error) + // Datasources lists all data sources (with elevated permissions). + Datasources(ctx context.Context) (*v0alpha1.DataSourceConnectionList, error) +} + +type DefaultQuerier struct { + connectionResourceInfo common.ResourceInfo + pluginJSON plugins.JSONData + pluginClient plugins.Client + dsService datasources.DataSourceService + dsCache datasources.CacheService +} + +func NewDefaultQuerier( + connectionResourceInfo common.ResourceInfo, + pluginJSON plugins.JSONData, + pluginClient plugins.Client, + dsService datasources.DataSourceService, + dsCache datasources.CacheService, +) *DefaultQuerier { + return &DefaultQuerier{ + connectionResourceInfo: connectionResourceInfo, + pluginJSON: pluginJSON, + pluginClient: pluginClient, + dsService: dsService, + dsCache: dsCache, + } +} + +func (q *DefaultQuerier) Query(ctx context.Context, query *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + _, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + return q.pluginClient.QueryData(ctx, query) +} + +func (q *DefaultQuerier) Resource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + _, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return err + } + return q.pluginClient.CallResource(ctx, req, sender) +} + +func (q *DefaultQuerier) Health(ctx context.Context, query *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { + _, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + return q.pluginClient.CheckHealth(ctx, query) +} + +func (q *DefaultQuerier) Datasource(ctx context.Context, name string) (*v0alpha1.DataSourceConnection, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + user, err := appcontext.User(ctx) + if err != nil { + return nil, err + } + ds, err := q.dsCache.GetDatasourceByUID(ctx, name, user, false) + if err != nil { + return nil, err + } + return asConnection(ds, info.Value) +} + +func (q *DefaultQuerier) Datasources(ctx context.Context) (*v0alpha1.DataSourceConnectionList, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + + ds, err := q.dsService.GetDataSourcesByType(ctx, &datasources.GetDataSourcesByTypeQuery{ + OrgID: info.OrgID, + Type: q.pluginJSON.ID, + }) + if err != nil { + return nil, err + } + return asConnectionList(q.connectionResourceInfo.TypeMeta(), ds, info.Value) +} + +func asConnectionList(typeMeta metav1.TypeMeta, dss []*datasources.DataSource, ns string) (*v0alpha1.DataSourceConnectionList, error) { + result := &v0alpha1.DataSourceConnectionList{ + Items: []v0alpha1.DataSourceConnection{}, + } + for _, ds := range dss { + v, _ := asConnection(ds, ns) + result.Items = append(result.Items, *v) + } + + return result, nil +} diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index 9fc43b7bc38..b5069a13cef 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -23,13 +23,13 @@ import ( query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" ) -var _ grafanaapiserver.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) // DataSourceAPIBuilder is used just so wire has something unique to return type DataSourceAPIBuilder struct { @@ -44,7 +44,7 @@ type DataSourceAPIBuilder struct { func RegisterAPIService( features featuremgmt.FeatureToggles, - apiRegistrar grafanaapiserver.APIRegistrar, + apiRegistrar builder.APIRegistrar, pluginClient plugins.Client, // access to everything datasources PluginDatasourceProvider, contextProvider PluginContextWrapper, @@ -151,6 +151,7 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { storage := map[string]rest.Storage{} @@ -213,6 +214,6 @@ func (b *DataSourceAPIBuilder) GetOpenAPIDefinitions() openapi.GetOpenAPIDefinit } // Register additional routes with the server -func (b *DataSourceAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *DataSourceAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil } diff --git a/pkg/registry/apis/datasource/standalone.go b/pkg/registry/apis/datasource/standalone.go index 34324bd11b8..fed03c18e13 100644 --- a/pkg/registry/apis/datasource/standalone.go +++ b/pkg/registry/apis/datasource/standalone.go @@ -10,8 +10,8 @@ import ( "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/accesscontrol/actest" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/setting" testdatasource "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource" ) diff --git a/pkg/registry/apis/example/dummy_storage.go b/pkg/registry/apis/example/dummy_storage.go index 290623771cb..ac4f823dd13 100644 --- a/pkg/registry/apis/example/dummy_storage.go +++ b/pkg/registry/apis/example/dummy_storage.go @@ -13,8 +13,8 @@ import ( "k8s.io/apiserver/pkg/registry/rest" example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" - grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" + grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" ) var ( diff --git a/pkg/registry/apis/example/register.go b/pkg/registry/apis/example/register.go index bd1277615a1..69a4add5d58 100644 --- a/pkg/registry/apis/example/register.go +++ b/pkg/registry/apis/example/register.go @@ -22,11 +22,11 @@ import ( example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" ) -var _ grafanaapiserver.APIGroupBuilder = (*TestingAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*TestingAPIBuilder)(nil) // This is used just so wire has something unique to return type TestingAPIBuilder struct { @@ -40,7 +40,7 @@ func NewTestingAPIBuilder() *TestingAPIBuilder { } } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration grafanaapiserver.APIRegistrar) *TestingAPIBuilder { +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *TestingAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis } @@ -84,7 +84,8 @@ func (b *TestingAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { func (b *TestingAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? - optsGetter generic.RESTOptionsGetter, + _ generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { b.codecs = codecs apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(b.gv.Group, scheme, metav1.ParameterCodec, codecs) @@ -102,9 +103,9 @@ func (b *TestingAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions } // Register additional routes with the server -func (b *TestingAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { - return &grafanaapiserver.APIRoutes{ - Root: []grafanaapiserver.APIRouteHandler{ +func (b *TestingAPIBuilder) GetAPIRoutes() *builder.APIRoutes { + return &builder.APIRoutes{ + Root: []builder.APIRouteHandler{ { Path: "aaa", Spec: &spec3.PathProps{ @@ -166,7 +167,7 @@ func (b *TestingAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { }, }, }, - Namespace: []grafanaapiserver.APIRouteHandler{ + Namespace: []builder.APIRouteHandler{ { Path: "ccc", Spec: &spec3.PathProps{ diff --git a/pkg/registry/apis/example/storage.go b/pkg/registry/apis/example/storage.go index fbec2ad7316..e2e198fa0be 100644 --- a/pkg/registry/apis/example/storage.go +++ b/pkg/registry/apis/example/storage.go @@ -12,7 +12,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/registry/apis/example/subresource.go b/pkg/registry/apis/example/subresource.go index d7d41186b54..63fdcab5f02 100644 --- a/pkg/registry/apis/example/subresource.go +++ b/pkg/registry/apis/example/subresource.go @@ -10,7 +10,7 @@ import ( example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" ) type dummySubresourceREST struct{} diff --git a/pkg/registry/apis/featuretoggle/features.go b/pkg/registry/apis/featuretoggle/features.go index b001d51d222..23808272b73 100644 --- a/pkg/registry/apis/featuretoggle/features.go +++ b/pkg/registry/apis/featuretoggle/features.go @@ -12,8 +12,8 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" ) var ( diff --git a/pkg/registry/apis/featuretoggle/register.go b/pkg/registry/apis/featuretoggle/register.go index 1a77fba7a3a..080bb42f20e 100644 --- a/pkg/registry/apis/featuretoggle/register.go +++ b/pkg/registry/apis/featuretoggle/register.go @@ -14,11 +14,11 @@ import ( "k8s.io/kube-openapi/pkg/validation/spec" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" ) -var _ grafanaapiserver.APIGroupBuilder = (*FeatureFlagAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*FeatureFlagAPIBuilder)(nil) var gv = v0alpha1.SchemeGroupVersion @@ -32,7 +32,7 @@ func NewFeatureFlagAPIBuilder(features *featuremgmt.FeatureManager) *FeatureFlag } func RegisterAPIService(features *featuremgmt.FeatureManager, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, ) *FeatureFlagAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis @@ -78,7 +78,8 @@ func (b *FeatureFlagAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { func (b *FeatureFlagAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? - optsGetter generic.RESTOptionsGetter, + _ generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -102,13 +103,13 @@ func (b *FeatureFlagAPIBuilder) GetAuthorizer() authorizer.Authorizer { } // Register additional routes with the server -func (b *FeatureFlagAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *FeatureFlagAPIBuilder) GetAPIRoutes() *builder.APIRoutes { defs := v0alpha1.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) stateSchema := defs["github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1.ResolvedToggleState"].Schema tags := []string{"Editor"} - return &grafanaapiserver.APIRoutes{ - Root: []grafanaapiserver.APIRouteHandler{ + return &builder.APIRoutes{ + Root: []builder.APIRouteHandler{ { Path: "current", Spec: &spec3.PathProps{ diff --git a/pkg/registry/apis/featuretoggle/toggles.go b/pkg/registry/apis/featuretoggle/toggles.go index b4671104c4a..81f5c092a3e 100644 --- a/pkg/registry/apis/featuretoggle/toggles.go +++ b/pkg/registry/apis/featuretoggle/toggles.go @@ -11,8 +11,8 @@ import ( common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) var ( diff --git a/pkg/registry/apis/folders/conversions.go b/pkg/registry/apis/folders/conversions.go index d14febaaaa9..11ec26f28fb 100644 --- a/pkg/registry/apis/folders/conversions.go +++ b/pkg/registry/apis/folders/conversions.go @@ -6,9 +6,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" ) func convertToK8sResource(v *folder.Folder, namespacer request.NamespaceMapper) *v0alpha1.Folder { diff --git a/pkg/registry/apis/folders/legacy_storage.go b/pkg/registry/apis/folders/legacy_storage.go index e6515b13beb..ac258217ccc 100644 --- a/pkg/registry/apis/folders/legacy_storage.go +++ b/pkg/registry/apis/folders/legacy_storage.go @@ -13,11 +13,11 @@ import ( "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/storage/entity" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/entity" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/util" ) diff --git a/pkg/registry/apis/folders/register.go b/pkg/registry/apis/folders/register.go index 4e385b2bded..15544b9f68b 100644 --- a/pkg/registry/apis/folders/register.go +++ b/pkg/registry/apis/folders/register.go @@ -14,16 +14,16 @@ import ( common "k8s.io/kube-openapi/pkg/common" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" "github.com/grafana/grafana/pkg/setting" ) -var _ grafanaapiserver.APIGroupBuilder = (*FolderAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*FolderAPIBuilder)(nil) var resourceInfo = v0alpha1.FolderResourceInfo @@ -37,7 +37,7 @@ type FolderAPIBuilder struct { func RegisterAPIService(cfg *setting.Cfg, features *featuremgmt.FeatureManager, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, folderSvc folder.Service, ) *FolderAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { @@ -89,6 +89,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -122,7 +123,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( storage[resourceInfo.StoragePath("children")] = &subChildrenREST{b.folderSvc} // enable dual writes if a RESTOptionsGetter is provided - if optsGetter != nil { + if dualWrite && optsGetter != nil { store, err := newStorage(scheme, optsGetter, legacyStore) if err != nil { return nil, err @@ -138,7 +139,7 @@ func (b *FolderAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions return v0alpha1.GetOpenAPIDefinitions } -func (b *FolderAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *FolderAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil // no custom API routes } diff --git a/pkg/registry/apis/folders/storage.go b/pkg/registry/apis/folders/storage.go index b3f7ee2d073..dc90e9ff792 100644 --- a/pkg/registry/apis/folders/storage.go +++ b/pkg/registry/apis/folders/storage.go @@ -6,8 +6,8 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" ) var _ grafanarest.Storage = (*storage)(nil) diff --git a/pkg/registry/apis/folders/sub_children.go b/pkg/registry/apis/folders/sub_children.go index 8e9be8b0813..9d5deb40712 100644 --- a/pkg/registry/apis/folders/sub_children.go +++ b/pkg/registry/apis/folders/sub_children.go @@ -9,8 +9,8 @@ import ( "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) type subChildrenREST struct { diff --git a/pkg/registry/apis/folders/sub_parents.go b/pkg/registry/apis/folders/sub_parents.go index 95ff32f42a4..36f67032d60 100644 --- a/pkg/registry/apis/folders/sub_parents.go +++ b/pkg/registry/apis/folders/sub_parents.go @@ -8,8 +8,8 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" ) type subParentsREST struct { diff --git a/pkg/registry/apis/playlist/conversions.go b/pkg/registry/apis/playlist/conversions.go index 43317aac080..c301cf9c04f 100644 --- a/pkg/registry/apis/playlist/conversions.go +++ b/pkg/registry/apis/playlist/conversions.go @@ -11,8 +11,8 @@ import ( "k8s.io/apimachinery/pkg/types" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" ) diff --git a/pkg/registry/apis/playlist/conversions_test.go b/pkg/registry/apis/playlist/conversions_test.go index 3bb8284bb8e..83188572030 100644 --- a/pkg/registry/apis/playlist/conversions_test.go +++ b/pkg/registry/apis/playlist/conversions_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/playlist" ) diff --git a/pkg/registry/apis/playlist/legacy_storage.go b/pkg/registry/apis/playlist/legacy_storage.go index 699d6e5371c..316c0f52462 100644 --- a/pkg/registry/apis/playlist/legacy_storage.go +++ b/pkg/registry/apis/playlist/legacy_storage.go @@ -11,7 +11,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" ) diff --git a/pkg/registry/apis/playlist/register.go b/pkg/registry/apis/playlist/register.go index b1dab9754f0..4bb5998ca9b 100644 --- a/pkg/registry/apis/playlist/register.go +++ b/pkg/registry/apis/playlist/register.go @@ -15,15 +15,15 @@ import ( common "k8s.io/kube-openapi/pkg/common" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/setting" ) -var _ grafanaapiserver.APIGroupBuilder = (*PlaylistAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*PlaylistAPIBuilder)(nil) // This is used just so wire has something unique to return type PlaylistAPIBuilder struct { @@ -33,7 +33,7 @@ type PlaylistAPIBuilder struct { } func RegisterAPIService(p playlistsvc.Service, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, cfg *setting.Cfg, ) *PlaylistAPIBuilder { builder := &PlaylistAPIBuilder{ @@ -79,6 +79,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(playlist.GROUP, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} @@ -128,7 +129,7 @@ func (b *PlaylistAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinition return playlist.GetOpenAPIDefinitions } -func (b *PlaylistAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *PlaylistAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil // no custom API routes } diff --git a/pkg/registry/apis/playlist/storage.go b/pkg/registry/apis/playlist/storage.go index b0f848cc11f..eb7512c717b 100644 --- a/pkg/registry/apis/playlist/storage.go +++ b/pkg/registry/apis/playlist/storage.go @@ -6,8 +6,8 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" ) var _ grafanarest.Storage = (*storage)(nil) diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go index 9b450c4904d..e1f3fa02cd2 100644 --- a/pkg/registry/apis/query/register.go +++ b/pkg/registry/apis/query/register.go @@ -21,14 +21,14 @@ import ( "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry/apis/query/runner" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" ) -var _ grafanaapiserver.APIGroupBuilder = (*QueryAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*QueryAPIBuilder)(nil) type QueryAPIBuilder struct { log log.Logger @@ -54,7 +54,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles, } func RegisterAPIService(features featuremgmt.FeatureToggles, - apiregistration grafanaapiserver.APIRegistrar, + apiregistration builder.APIRegistrar, dataSourcesService datasources.DataSourceService, pluginStore pluginstore.Store, accessControl accesscontrol.AccessControl, @@ -106,6 +106,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { gv := v0alpha1.SchemeGroupVersion apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs) @@ -124,7 +125,7 @@ func (b *QueryAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions { } // Register additional routes with the server -func (b *QueryAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *QueryAPIBuilder) GetAPIRoutes() *builder.APIRoutes { defs := v0alpha1.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) querySchema := defs["github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryRequest"].Schema responseSchema := defs["github.com/grafana/grafana/pkg/apis/query/v0alpha1.QueryDataResponse"].Schema @@ -167,9 +168,9 @@ func (b *QueryAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { "to": "1704914981544" }`), &randomWalkTable) - return &grafanaapiserver.APIRoutes{ - Root: []grafanaapiserver.APIRouteHandler{}, - Namespace: []grafanaapiserver.APIRouteHandler{ + return &builder.APIRoutes{ + Root: []builder.APIRouteHandler{}, + Namespace: []builder.APIRouteHandler{ { Path: "query", Spec: &spec3.PathProps{ diff --git a/pkg/registry/apis/service/register.go b/pkg/registry/apis/service/register.go index c2cc3e57584..f7366b3badf 100644 --- a/pkg/registry/apis/service/register.go +++ b/pkg/registry/apis/service/register.go @@ -12,11 +12,11 @@ import ( "k8s.io/kube-openapi/pkg/common" service "github.com/grafana/grafana/pkg/apis/service/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" ) -var _ grafanaapiserver.APIGroupBuilder = (*ServiceAPIBuilder)(nil) +var _ builder.APIGroupBuilder = (*ServiceAPIBuilder)(nil) // This is used just so wire has something unique to return type ServiceAPIBuilder struct{} @@ -25,10 +25,7 @@ func NewServiceAPIBuilder() *ServiceAPIBuilder { return &ServiceAPIBuilder{} } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration grafanaapiserver.APIRegistrar) *ServiceAPIBuilder { - if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { - return nil // skip registration unless opting into experimental apis - } +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ServiceAPIBuilder { builder := NewServiceAPIBuilder() apiregistration.RegisterAPI(NewServiceAPIBuilder()) return builder @@ -64,6 +61,7 @@ func (b *ServiceAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, + _ bool, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(service.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -83,6 +81,6 @@ func (b *ServiceAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions } // Register additional routes with the server -func (b *ServiceAPIBuilder) GetAPIRoutes() *grafanaapiserver.APIRoutes { +func (b *ServiceAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil } diff --git a/pkg/registry/apis/service/storage.go b/pkg/registry/apis/service/storage.go index db4dc56daf3..bf0d8551b80 100644 --- a/pkg/registry/apis/service/storage.go +++ b/pkg/registry/apis/service/storage.go @@ -10,9 +10,9 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" service "github.com/grafana/grafana/pkg/apis/service/v0alpha1" - grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + grafanaregistry "github.com/grafana/grafana/pkg/services/apiserver/registry/generic" + grafanarest "github.com/grafana/grafana/pkg/services/apiserver/rest" + "github.com/grafana/grafana/pkg/services/apiserver/utils" ) var _ grafanarest.Storage = (*storage)(nil) diff --git a/pkg/registry/apis/wireset.go b/pkg/registry/apis/wireset.go index c5da75bf909..e093f7a175e 100644 --- a/pkg/registry/apis/wireset.go +++ b/pkg/registry/apis/wireset.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/folders" "github.com/grafana/grafana/pkg/registry/apis/playlist" "github.com/grafana/grafana/pkg/registry/apis/query" + "github.com/grafana/grafana/pkg/registry/apis/service" "github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext" ) @@ -28,5 +29,6 @@ var WireSet = wire.NewSet( featuretoggle.RegisterAPIService, datasource.RegisterAPIService, folders.RegisterAPIService, + service.RegisterAPIService, query.RegisterAPIService, ) diff --git a/pkg/registry/backgroundsvcs/background_services.go b/pkg/registry/backgroundsvcs/background_services.go index 7c8a0821833..3f323ab09ae 100644 --- a/pkg/registry/backgroundsvcs/background_services.go +++ b/pkg/registry/backgroundsvcs/background_services.go @@ -11,10 +11,10 @@ import ( apiregistry "github.com/grafana/grafana/pkg/registry/apis" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/anonymous/anonimpl" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/cleanup" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" "github.com/grafana/grafana/pkg/services/grpcserver" "github.com/grafana/grafana/pkg/services/guardian" ldapapi "github.com/grafana/grafana/pkg/services/ldap/api" diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 65458a41d2f..6e7c1629129 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -44,6 +44,7 @@ import ( "github.com/grafana/grafana/pkg/services/annotations/annotationsimpl" "github.com/grafana/grafana/pkg/services/anonymous/anonimpl/anonstore" "github.com/grafana/grafana/pkg/services/apikey/apikeyimpl" + grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/auth/idimpl" "github.com/grafana/grafana/pkg/services/auth/jwt" @@ -72,7 +73,6 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" "github.com/grafana/grafana/pkg/services/folder/folderimpl" - grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver" "github.com/grafana/grafana/pkg/services/grpcserver" grpccontext "github.com/grafana/grafana/pkg/services/grpcserver/context" "github.com/grafana/grafana/pkg/services/grpcserver/interceptors" diff --git a/pkg/services/grafana-apiserver/README.md b/pkg/services/apiserver/README.md similarity index 100% rename from pkg/services/grafana-apiserver/README.md rename to pkg/services/apiserver/README.md diff --git a/pkg/aggregator/README.md b/pkg/services/apiserver/aggregator/README.md similarity index 100% rename from pkg/aggregator/README.md rename to pkg/services/apiserver/aggregator/README.md diff --git a/pkg/services/apiserver/aggregator/aggregator.go b/pkg/services/apiserver/aggregator/aggregator.go new file mode 100644 index 00000000000..b62e106446e --- /dev/null +++ b/pkg/services/apiserver/aggregator/aggregator.go @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/aggregator.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/apiserver/apiextensions.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. + +package aggregator + +import ( + "crypto/tls" + "fmt" + "net/http" + "strings" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/sets" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + apiregistrationclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" + apiregistrationInformers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" + "k8s.io/kube-aggregator/pkg/controllers/autoregister" + + serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned" + informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions" + "github.com/grafana/grafana/pkg/services/apiserver/options" +) + +func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig genericapiserver.RecommendedConfig) (*aggregatorapiserver.Config, informersv0alpha1.SharedInformerFactory, error) { + // Create a fake clientset and informers for the k8s v1 API group. + // These are not used in grafana's aggregator because v1 APIs are not available. + fakev1Informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute) + + serviceClient, err := serviceclientset.NewForConfig(sharedConfig.LoopbackClientConfig) + if err != nil { + return nil, nil, err + } + sharedInformerFactory := informersv0alpha1.NewSharedInformerFactory( + serviceClient, + 5*time.Minute, // this is effectively used as a refresh interval right now. Might want to do something nicer later on. + ) + serviceResolver := NewExternalNameResolver(sharedInformerFactory.Service().V0alpha1().ExternalNames().Lister()) + + aggregatorConfig := &aggregatorapiserver.Config{ + GenericConfig: &genericapiserver.RecommendedConfig{ + Config: sharedConfig.Config, + SharedInformerFactory: fakev1Informers, + ClientConfig: sharedConfig.LoopbackClientConfig, + }, + ExtraConfig: aggregatorapiserver.ExtraConfig{ + ProxyClientCertFile: commandOptions.AggregatorOptions.ProxyClientCertFile, + ProxyClientKeyFile: commandOptions.AggregatorOptions.ProxyClientKeyFile, + // NOTE: while ProxyTransport can be skipped in the configuration, it allows honoring + // DISABLE_HTTP2, HTTPS_PROXY and NO_PROXY env vars as needed + ProxyTransport: createProxyTransport(), + ServiceResolver: serviceResolver, + }, + } + + if err := commandOptions.AggregatorOptions.ApplyTo(aggregatorConfig, commandOptions.RecommendedOptions.Etcd, commandOptions.StorageOptions.DataPath); err != nil { + return nil, nil, err + } + + return aggregatorConfig, sharedInformerFactory, nil +} + +func CreateAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, sharedInformerFactory informersv0alpha1.SharedInformerFactory, delegateAPIServer genericapiserver.DelegationTarget) (*aggregatorapiserver.APIAggregator, error) { + completedConfig := aggregatorConfig.Complete() + aggregatorServer, err := completedConfig.NewWithDelegate(delegateAPIServer) + if err != nil { + return nil, err + } + + // create controllers for auto-registration + apiRegistrationClient, err := apiregistrationclient.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) + if err != nil { + return nil, err + } + + autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient) + apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController) + + // Imbue all builtin group-priorities onto the aggregated discovery + if completedConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil { + for gv, entry := range APIVersionPriorities { + completedConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.Group), int(entry.Version)) + } + } + + err = aggregatorServer.GenericAPIServer.AddPostStartHook("grafana-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error { + go func() { + autoRegistrationController.Run(5, context.StopCh) + }() + return nil + }) + if err != nil { + return nil, err + } + + err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks( + makeAPIServiceAvailableHealthCheck( + "autoregister-completion", + apiServices, + aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), + ), + ) + if err != nil { + return nil, err + } + + apiregistrationClient, err := apiregistrationclientset.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig) + if err != nil { + return nil, err + } + + availableController, err := NewAvailableConditionController( + aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), + sharedInformerFactory.Service().V0alpha1().ExternalNames(), + apiregistrationClient.ApiregistrationV1(), + nil, + (func() ([]byte, []byte))(nil), + completedConfig.ExtraConfig.ServiceResolver, + ) + if err != nil { + return nil, err + } + + aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-override-available-controller", func(context genericapiserver.PostStartHookContext) error { + // if we end up blocking for long periods of time, we may need to increase workers. + go availableController.Run(5, context.StopCh) + return nil + }) + + aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("start-grafana-aggregator-informers", func(context genericapiserver.PostStartHookContext) error { + sharedInformerFactory.Start(context.StopCh) + aggregatorServer.APIRegistrationInformers.Start(context.StopCh) + return nil + }) + + return aggregatorServer, nil +} + +func makeAPIService(gv schema.GroupVersion) *v1.APIService { + apiServicePriority, ok := APIVersionPriorities[gv] + if !ok { + // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version + // being permanently stuck in the APIServices list. + klog.Infof("Skipping APIService creation for %v", gv) + return nil + } + return &v1.APIService{ + ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, + Spec: v1.APIServiceSpec{ + Group: gv.Group, + Version: gv.Version, + GroupPriorityMinimum: apiServicePriority.Group, + VersionPriority: apiServicePriority.Version, + }, + } +} + +// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy +// once all of the specified services have been observed to be available at least once. +func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer apiregistrationInformers.APIServiceInformer) healthz.HealthChecker { + // Track the auto-registered API services that have not been observed to be available yet + pendingServiceNamesLock := &sync.RWMutex{} + pendingServiceNames := sets.NewString() + for _, service := range apiServices { + pendingServiceNames.Insert(service.Name) + } + + // When an APIService in the list is seen as available, remove it from the pending list + handleAPIServiceChange := func(service *v1.APIService) { + pendingServiceNamesLock.Lock() + defer pendingServiceNamesLock.Unlock() + if !pendingServiceNames.Has(service.Name) { + return + } + if v1helper.IsAPIServiceConditionTrue(service, v1.Available) { + pendingServiceNames.Delete(service.Name) + } + } + + // Watch add/update events for APIServices + _, _ = apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) }, + UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) }, + }) + + // Don't return healthy until the pending list is empty + return healthz.NamedCheck(name, func(r *http.Request) error { + pendingServiceNamesLock.RLock() + defer pendingServiceNamesLock.RUnlock() + if pendingServiceNames.Len() > 0 { + return fmt.Errorf("missing APIService: %v", pendingServiceNames.List()) + } + return nil + }) +} + +// Priority defines group Priority that is used in discovery. This controls +// group position in the kubectl output. +type Priority struct { + // Group indicates the order of the Group relative to other groups. + Group int32 + // Version indicates the relative order of the Version inside of its group. + Version int32 +} + +// APIVersionPriorities are the proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers +// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. +// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. +// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. +var APIVersionPriorities = map[schema.GroupVersion]Priority{ + {Group: "", Version: "v1"}: {Group: 18000, Version: 1}, + // to my knowledge, nothing below here collides + {Group: "admissionregistration.k8s.io", Version: "v1"}: {Group: 16700, Version: 15}, + {Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {Group: 16700, Version: 12}, + {Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {Group: 16700, Version: 9}, + // Append a new group to the end of the list if unsure. + // You can use min(existing group)-100 as the initial value for a group. + // Version can be set to 9 (to have space around) for a new group. +} + +func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService { + apiServices := []*v1.APIService{} + + for _, curr := range delegateAPIServer.ListedPaths() { + if curr == "/api/v1" { + apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"}) + registration.AddAPIServiceToSyncOnStart(apiService) + apiServices = append(apiServices, apiService) + continue + } + + if !strings.HasPrefix(curr, "/apis/") { + continue + } + // this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1 + tokens := strings.Split(curr, "/") + if len(tokens) != 4 { + continue + } + + apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) + if apiService == nil { + continue + } + registration.AddAPIServiceToSyncOnStart(apiService) + apiServices = append(apiServices, apiService) + } + + return apiServices +} + +// NOTE: below function imported from https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L197 +// createProxyTransport creates the dialer infrastructure to connect to the api servers. +func createProxyTransport() *http.Transport { + // NOTE: We don't set proxyDialerFn but the below SetTransportDefaults will + // See https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/net/http.go#L109 + var proxyDialerFn utilnet.DialFunc + // Proxying to services is IP-based... don't expect to be able to verify the hostname + proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} + proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ + DialContext: proxyDialerFn, + TLSClientConfig: proxyTLSClientConfig, + }) + return proxyTransport +} diff --git a/pkg/aggregator/availableController.go b/pkg/services/apiserver/aggregator/availableController.go similarity index 100% rename from pkg/aggregator/availableController.go rename to pkg/services/apiserver/aggregator/availableController.go diff --git a/pkg/aggregator/resolver.go b/pkg/services/apiserver/aggregator/resolver.go similarity index 100% rename from pkg/aggregator/resolver.go rename to pkg/services/apiserver/aggregator/resolver.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/impersonation.go b/pkg/services/apiserver/auth/authorizer/impersonation.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/impersonation.go rename to pkg/services/apiserver/auth/authorizer/impersonation.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/impersonation_test.go b/pkg/services/apiserver/auth/authorizer/impersonation_test.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/impersonation_test.go rename to pkg/services/apiserver/auth/authorizer/impersonation_test.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/org_id.go b/pkg/services/apiserver/auth/authorizer/org_id.go similarity index 95% rename from pkg/services/grafana-apiserver/auth/authorizer/org_id.go rename to pkg/services/apiserver/auth/authorizer/org_id.go index 4d6efdfc88a..e9d4a6786a8 100644 --- a/pkg/services/grafana-apiserver/auth/authorizer/org_id.go +++ b/pkg/services/apiserver/auth/authorizer/org_id.go @@ -8,7 +8,7 @@ import ( "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/log" - grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/org" ) diff --git a/pkg/services/grafana-apiserver/auth/authorizer/org_role.go b/pkg/services/apiserver/auth/authorizer/org_role.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/org_role.go rename to pkg/services/apiserver/auth/authorizer/org_role.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/provider.go b/pkg/services/apiserver/auth/authorizer/provider.go similarity index 100% rename from pkg/services/grafana-apiserver/auth/authorizer/provider.go rename to pkg/services/apiserver/auth/authorizer/provider.go diff --git a/pkg/services/grafana-apiserver/auth/authorizer/stack_id.go b/pkg/services/apiserver/auth/authorizer/stack_id.go similarity index 94% rename from pkg/services/grafana-apiserver/auth/authorizer/stack_id.go rename to pkg/services/apiserver/auth/authorizer/stack_id.go index 32887dd8142..0098c1dfd21 100644 --- a/pkg/services/grafana-apiserver/auth/authorizer/stack_id.go +++ b/pkg/services/apiserver/auth/authorizer/stack_id.go @@ -8,7 +8,7 @@ import ( "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/log" - grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/services/grafana-apiserver/common.go b/pkg/services/apiserver/builder/common.go similarity index 94% rename from pkg/services/grafana-apiserver/common.go rename to pkg/services/apiserver/builder/common.go index 5845bfc3c22..39d2c5dccee 100644 --- a/pkg/services/grafana-apiserver/common.go +++ b/pkg/services/apiserver/builder/common.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package builder import ( "net/http" @@ -27,6 +27,7 @@ type APIGroupBuilder interface { scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, + dualWrite bool, ) (*genericapiserver.APIGroupInfo, error) // Get OpenAPI definitions @@ -57,3 +58,7 @@ type APIRoutes struct { // Namespace handlers are mounted under the namespace Namespace []APIRouteHandler } + +type APIRegistrar interface { + RegisterAPI(builder APIGroupBuilder) +} diff --git a/pkg/services/grafana-apiserver/helper.go b/pkg/services/apiserver/builder/helper.go similarity index 65% rename from pkg/services/grafana-apiserver/helper.go rename to pkg/services/apiserver/builder/helper.go index 8205715487f..f6987752365 100644 --- a/pkg/services/grafana-apiserver/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -1,34 +1,45 @@ -package grafanaapiserver +package builder import ( "fmt" "net/http" goruntime "runtime" + "runtime/debug" + "strconv" "strings" "time" + "golang.org/x/mod/semver" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/version" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/util/openapi" - "k8s.io/client-go/kubernetes/scheme" + k8sscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/kube-openapi/pkg/common" "github.com/grafana/grafana/pkg/setting" ) -func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []APIGroupBuilder) error { +func SetupConfig( + scheme *runtime.Scheme, + serverConfig *genericapiserver.RecommendedConfig, + builders []APIGroupBuilder, +) error { defsGetter := GetOpenAPIDefinitions(builders) serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig( openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter), - openapinamer.NewDefinitionNamer(Scheme, scheme.Scheme)) + openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme)) serverConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config( openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter), - openapinamer.NewDefinitionNamer(Scheme, scheme.Scheme)) + openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme)) + // Add the custom routes to service discovery + serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(builders) serverConfig.OpenAPIV3Config.GetOperationIDAndTagsFromRoute = func(r common.Route) (string, []string, error) { tags := []string{} prop, ok := r.Metadata()["x-kubernetes-group-version-kind"] @@ -41,9 +52,6 @@ func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []AP return r.OperationName(), tags, nil } - // Add the custom routes to service discovery - serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(builders) - // Set the swagger build versions serverConfig.OpenAPIConfig.Info.Version = setting.BuildVersion serverConfig.OpenAPIV3Config.Info.Version = setting.BuildVersion @@ -82,12 +90,16 @@ func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []AP return nil } -func InstallAPIs(server *genericapiserver.GenericAPIServer, +func InstallAPIs( + scheme *runtime.Scheme, + codecs serializer.CodecFactory, + server *genericapiserver.GenericAPIServer, optsGetter generic.RESTOptionsGetter, builders []APIGroupBuilder, + dualWrite bool, ) error { for _, b := range builders { - g, err := b.GetAPIGroupInfo(Scheme, Codecs, optsGetter) + g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, dualWrite) if err != nil { return err } @@ -101,3 +113,33 @@ func InstallAPIs(server *genericapiserver.GenericAPIServer, } return nil } + +// find the k8s version according to build info +func getK8sApiserverVersion() (string, error) { + bi, ok := debug.ReadBuildInfo() + if !ok { + return "", fmt.Errorf("debug.ReadBuildInfo() failed") + } + + if len(bi.Deps) == 0 { + return "v?.?", nil // this is normal while debugging + } + + for _, dep := range bi.Deps { + if dep.Path == "k8s.io/apiserver" { + if !semver.IsValid(dep.Version) { + return "", fmt.Errorf("invalid semantic version for k8s.io/apiserver") + } + // v0 => v1 + majorVersion := strings.TrimPrefix(semver.Major(dep.Version), "v") + majorInt, err := strconv.Atoi(majorVersion) + if err != nil { + return "", fmt.Errorf("could not convert majorVersion to int. majorVersion: %s", majorVersion) + } + newMajor := fmt.Sprintf("v%d", majorInt+1) + return strings.Replace(dep.Version, semver.Major(dep.Version), newMajor, 1), nil + } + } + + return "", fmt.Errorf("could not find k8s.io/apiserver in build info") +} diff --git a/pkg/services/grafana-apiserver/openapi.go b/pkg/services/apiserver/builder/openapi.go similarity index 99% rename from pkg/services/grafana-apiserver/openapi.go rename to pkg/services/apiserver/builder/openapi.go index f90b6bfdc46..e4d6a2ac149 100644 --- a/pkg/services/grafana-apiserver/openapi.go +++ b/pkg/services/apiserver/builder/openapi.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package builder import ( "maps" @@ -15,8 +15,8 @@ func GetOpenAPIDefinitions(builders []APIGroupBuilder) common.GetOpenAPIDefiniti return func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { defs := getStandardOpenAPIDefinitions(ref) maps.Copy(defs, v0alpha1.GetOpenAPIDefinitions(ref)) // common grafana apis - for _, builder := range builders { - g := builder.GetOpenAPIDefinitions() + for _, b := range builders { + g := b.GetOpenAPIDefinitions() if g != nil { out := g(ref) maps.Copy(defs, out) diff --git a/pkg/services/grafana-apiserver/request_handler.go b/pkg/services/apiserver/builder/request_handler.go similarity index 96% rename from pkg/services/grafana-apiserver/request_handler.go rename to pkg/services/apiserver/builder/request_handler.go index 48199878a67..b88c221b57f 100644 --- a/pkg/services/grafana-apiserver/request_handler.go +++ b/pkg/services/apiserver/builder/request_handler.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package builder import ( "fmt" @@ -125,9 +125,9 @@ func getOpenAPIPostProcessor(builders []APIGroupBuilder) func(*spec3.OpenAPI) (* if s.Paths == nil { return s, nil } - for _, builder := range builders { - routes := builder.GetAPIRoutes() - gv := builder.GetGroupVersion() + for _, b := range builders { + routes := b.GetAPIRoutes() + gv := b.GetGroupVersion() prefix := "/apis/" + gv.String() + "/" if s.Paths.Paths[prefix] != nil { copy := spec3.OpenAPI{ diff --git a/pkg/services/apiserver/config.go b/pkg/services/apiserver/config.go new file mode 100644 index 00000000000..3cc6fc82d07 --- /dev/null +++ b/pkg/services/apiserver/config.go @@ -0,0 +1,48 @@ +package apiserver + +import ( + "fmt" + "net" + "path/filepath" + "strconv" + + "github.com/grafana/grafana/pkg/services/apiserver/options" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" +) + +func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o *options.Options) { + defaultLogLevel := 0 + ip := net.ParseIP(cfg.HTTPAddr) + apiURL := cfg.AppURL + port, err := strconv.Atoi(cfg.HTTPPort) + if err != nil { + port = 3000 + } + + if cfg.Env == setting.Dev { + defaultLogLevel = 10 + port = 6443 + ip = net.ParseIP("127.0.0.1") + apiURL = fmt.Sprintf("https://%s:%d", ip, port) + } + + host := fmt.Sprintf("%s:%d", ip, port) + + o.RecommendedOptions.Etcd.StorageConfig.Transport.ServerList = cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(",") + + o.RecommendedOptions.SecureServing.BindAddress = ip + o.RecommendedOptions.SecureServing.BindPort = port + o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true + o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true + + o.RecommendedOptions.Admission = nil + o.RecommendedOptions.CoreAPI = nil + + o.StorageOptions.StorageType = options.StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(options.StorageTypeLegacy))) + o.StorageOptions.DataPath = filepath.Join(cfg.DataPath, "grafana-apiserver") + o.ExtraOptions.DevMode = features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess) + o.ExtraOptions.ExternalAddress = host + o.ExtraOptions.APIURL = apiURL + o.ExtraOptions.Verbosity = defaultLogLevel +} diff --git a/pkg/services/grafana-apiserver/endpoints/request/namespace.go b/pkg/services/apiserver/endpoints/request/namespace.go similarity index 100% rename from pkg/services/grafana-apiserver/endpoints/request/namespace.go rename to pkg/services/apiserver/endpoints/request/namespace.go diff --git a/pkg/services/grafana-apiserver/endpoints/request/namespace_test.go b/pkg/services/apiserver/endpoints/request/namespace_test.go similarity index 97% rename from pkg/services/grafana-apiserver/endpoints/request/namespace_test.go rename to pkg/services/apiserver/endpoints/request/namespace_test.go index 4f9dc7e9281..cd5a8e58394 100644 --- a/pkg/services/grafana-apiserver/endpoints/request/namespace_test.go +++ b/pkg/services/apiserver/endpoints/request/namespace_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/setting" ) diff --git a/pkg/services/apiserver/options/aggregator.go b/pkg/services/apiserver/options/aggregator.go new file mode 100644 index 00000000000..51c67cea117 --- /dev/null +++ b/pkg/services/apiserver/options/aggregator.go @@ -0,0 +1,112 @@ +package options + +import ( + "github.com/spf13/pflag" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" + genericfeatures "k8s.io/apiserver/pkg/features" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/server/resourceconfig" + utilfeature "k8s.io/apiserver/pkg/util/feature" + apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + aggregatoropenapi "k8s.io/kube-aggregator/pkg/generated/openapi" + "k8s.io/kube-openapi/pkg/common" + + servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1" + filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file" +) + +// AggregatorServerOptions contains the state for the aggregator apiserver +type AggregatorServerOptions struct { + AlternateDNS []string + ProxyClientCertFile string + ProxyClientKeyFile string +} + +func NewAggregatorServerOptions() *AggregatorServerOptions { + return &AggregatorServerOptions{} +} + +func (o *AggregatorServerOptions) getMergedOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + aggregatorAPIs := aggregatoropenapi.GetOpenAPIDefinitions(ref) + return aggregatorAPIs +} + +func (o *AggregatorServerOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, + "path to proxy client cert file") + + fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, + "path to proxy client cert file") +} + +func (o *AggregatorServerOptions) Validate() []error { + if o == nil { + return nil + } + + // TODO: do we need to validate anything here? + return nil +} + +func (o *AggregatorServerOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Config, etcdOpts *options.EtcdOptions, dataPath string) error { + genericConfig := aggregatorConfig.GenericConfig + + genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} + genericConfig.RESTOptionsGetter = nil + + if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) && + utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { + // Add StorageVersionPrecondition handler to aggregator-apiserver. + // The handler will block write requests to built-in resources until the + // target resources' storage versions are up-to-date. + genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition + } + + // copy the etcd options so we don't mutate originals. + // we assume that the etcd options have been completed already. avoid messing with anything outside + // of changes to StorageConfig as that may lead to unexpected behavior when the options are applied. + etcdOptions := *etcdOpts + etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion, + apiregistrationv1beta1.SchemeGroupVersion, + servicev0alpha1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion, + schema.GroupKind{Group: apiregistrationv1beta1.GroupName}, + schema.GroupKind{Group: servicev0alpha1.GROUP}) + etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks + if err := etcdOptions.ApplyTo(&genericConfig.Config); err != nil { + return err + } + // override the RESTOptionsGetter to use the file storage options getter + aggregatorConfig.GenericConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(dataPath, etcdOptions.StorageConfig) + + // prevent generic API server from installing the OpenAPI handler. Aggregator server has its own customized OpenAPI handler. + genericConfig.SkipOpenAPIInstallation = true + mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(aggregatorapiserver.DefaultAPIResourceConfigSource(), nil, aggregatorscheme.Scheme) + if err != nil { + return err + } + genericConfig.MergedResourceConfig = mergedResourceConfig + + namer := openapinamer.NewDefinitionNamer(aggregatorscheme.Scheme) + genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(o.getMergedOpenAPIDefinitions, namer) + genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes" + genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(o.getMergedOpenAPIDefinitions, namer) + genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" + genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} + + // These hooks use v1 informers, which are not available in the grafana aggregator. + genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("apiservice-status-available-controller") + genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("start-kube-aggregator-informers") + + return nil +} diff --git a/pkg/services/apiserver/options/extra.go b/pkg/services/apiserver/options/extra.go new file mode 100644 index 00000000000..fdb394d4d28 --- /dev/null +++ b/pkg/services/apiserver/options/extra.go @@ -0,0 +1,46 @@ +package options + +import ( + "strconv" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" +) + +type ExtraOptions struct { + DevMode bool + ExternalAddress string + APIURL string + Verbosity int +} + +func NewExtraOptions() *ExtraOptions { + return &ExtraOptions{ + DevMode: false, + Verbosity: 0, + } +} + +func (o *ExtraOptions) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.DevMode, "grafana-apiserver-dev-mode", o.DevMode, "Enable dev mode") + fs.StringVar(&o.ExternalAddress, "grafana-apiserver-host", o.ExternalAddress, "Host") + fs.StringVar(&o.APIURL, "grafana-apiserver-api-url", o.APIURL, "API URL") + fs.IntVar(&o.Verbosity, "verbosity", o.Verbosity, "Verbosity") +} + +func (o *ExtraOptions) Validate() []error { + return nil +} + +func (o *ExtraOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error { + logger := logr.New(newLogAdapter(o.Verbosity)) + klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true)) + if _, err := logs.GlogSetter(strconv.Itoa(o.Verbosity)); err != nil { + logger.Error(err, "failed to set log level") + } + c.ExternalAddress = o.ExternalAddress + return nil +} diff --git a/pkg/services/grafana-apiserver/log.go b/pkg/services/apiserver/options/log.go similarity index 97% rename from pkg/services/grafana-apiserver/log.go rename to pkg/services/apiserver/options/log.go index edf10c0d3d3..9e32f66e3b3 100644 --- a/pkg/services/grafana-apiserver/log.go +++ b/pkg/services/apiserver/options/log.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package options import ( "strings" diff --git a/pkg/services/apiserver/options/options.go b/pkg/services/apiserver/options/options.go new file mode 100644 index 00000000000..10c320e8925 --- /dev/null +++ b/pkg/services/apiserver/options/options.go @@ -0,0 +1,126 @@ +package options + +import ( + "net" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" + genericapiserver "k8s.io/apiserver/pkg/server" + genericoptions "k8s.io/apiserver/pkg/server/options" +) + +const defaultEtcdPathPrefix = "/registry/grafana.app" + +type Options struct { + RecommendedOptions *genericoptions.RecommendedOptions + AggregatorOptions *AggregatorServerOptions + StorageOptions *StorageOptions + ExtraOptions *ExtraOptions +} + +func NewOptions(codec runtime.Codec) *Options { + return &Options{ + RecommendedOptions: genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + codec, + ), + AggregatorOptions: NewAggregatorServerOptions(), + StorageOptions: NewStorageOptions(), + ExtraOptions: NewExtraOptions(), + } +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + o.RecommendedOptions.AddFlags(fs) + o.AggregatorOptions.AddFlags(fs) + o.StorageOptions.AddFlags(fs) + o.ExtraOptions.AddFlags(fs) +} + +func (o *Options) Validate() []error { + if errs := o.ExtraOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.StorageOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.AggregatorOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.RecommendedOptions.SecureServing.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.RecommendedOptions.Authentication.Validate(); len(errs) != 0 { + return errs + } + + if o.StorageOptions.StorageType == StorageTypeEtcd { + if errs := o.RecommendedOptions.Etcd.Validate(); len(errs) != 0 { + return errs + } + } + + return nil +} + +func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error { + serverConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis") + + if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil { + return err + } + + if !o.ExtraOptions.DevMode { + o.RecommendedOptions.SecureServing.Listener = newFakeListener() + } + + if err := o.RecommendedOptions.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { + return err + } + + if err := o.RecommendedOptions.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil { + return err + } + + if !o.ExtraOptions.DevMode { + if err := serverConfig.SecureServing.Listener.Close(); err != nil { + return err + } + serverConfig.SecureServing = nil + } + + return nil +} + +type fakeListener struct { + server net.Conn + client net.Conn +} + +func newFakeListener() *fakeListener { + server, client := net.Pipe() + return &fakeListener{ + server: server, + client: client, + } +} + +func (f *fakeListener) Accept() (net.Conn, error) { + return f.server, nil +} + +func (f *fakeListener) Close() error { + if err := f.client.Close(); err != nil { + return err + } + return f.server.Close() +} + +func (f *fakeListener) Addr() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 3000, Zone: ""} +} diff --git a/pkg/services/apiserver/options/storage.go b/pkg/services/apiserver/options/storage.go new file mode 100644 index 00000000000..51ccb84efe9 --- /dev/null +++ b/pkg/services/apiserver/options/storage.go @@ -0,0 +1,51 @@ +package options + +import ( + "fmt" + + "github.com/spf13/pflag" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" +) + +type StorageType string + +const ( + StorageTypeFile StorageType = "file" + StorageTypeEtcd StorageType = "etcd" + StorageTypeLegacy StorageType = "legacy" + StorageTypeUnified StorageType = "unified" + StorageTypeUnifiedGrpc StorageType = "unified-grpc" +) + +type StorageOptions struct { + StorageType StorageType + DataPath string +} + +func NewStorageOptions() *StorageOptions { + return &StorageOptions{ + StorageType: StorageTypeLegacy, + } +} + +func (o *StorageOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar((*string)(&o.StorageType), "grafana-apiserver-storage-type", string(o.StorageType), "Storage type") + fs.StringVar((*string)(&o.StorageType), "grafana-apiserver-storage-path", string(o.StorageType), "Storage path for file storage") +} + +func (o *StorageOptions) Validate() []error { + errs := []error{} + switch o.StorageType { + case StorageTypeFile, StorageTypeEtcd, StorageTypeLegacy, StorageTypeUnified, StorageTypeUnifiedGrpc: + // no-op + default: + errs = append(errs, fmt.Errorf("--grafana-apiserver-storage-type must be one of %s, %s, %s, %s, %s", StorageTypeFile, StorageTypeEtcd, StorageTypeLegacy, StorageTypeUnified, StorageTypeUnifiedGrpc)) + } + return errs +} + +func (o *StorageOptions) ApplyTo(serverConfig *genericapiserver.RecommendedConfig, etcdOptions *options.EtcdOptions) error { + // TODO: move storage setup here + return nil +} diff --git a/pkg/services/grafana-apiserver/registry/generic/strategy.go b/pkg/services/apiserver/registry/generic/strategy.go similarity index 100% rename from pkg/services/grafana-apiserver/registry/generic/strategy.go rename to pkg/services/apiserver/registry/generic/strategy.go diff --git a/pkg/services/grafana-apiserver/rest/dualwriter.go b/pkg/services/apiserver/rest/dualwriter.go similarity index 100% rename from pkg/services/grafana-apiserver/rest/dualwriter.go rename to pkg/services/apiserver/rest/dualwriter.go diff --git a/pkg/services/grafana-apiserver/service.go b/pkg/services/apiserver/service.go similarity index 63% rename from pkg/services/grafana-apiserver/service.go rename to pkg/services/apiserver/service.go index 84447d9db2f..3f47779488c 100644 --- a/pkg/services/grafana-apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -1,4 +1,4 @@ -package grafanaapiserver +package apiserver import ( "context" @@ -6,13 +6,8 @@ import ( "net/http" "net/http/httptest" "path" - "runtime/debug" - "strconv" - "strings" - "github.com/go-logr/logr" "github.com/grafana/dskit/services" - "golang.org/x/mod/semver" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,15 +16,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/endpoints/responsewriter" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" clientrest "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/component-base/logs" - "k8s.io/klog/v2" - - "github.com/grafana/grafana/pkg/services/grafana-apiserver/auth/authorizer" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" - "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/appcontext" @@ -38,26 +26,21 @@ import ( "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/modules" "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + grafanaapiserveroptions "github.com/grafana/grafana/pkg/services/apiserver/options" + entitystorage "github.com/grafana/grafana/pkg/services/apiserver/storage/entity" + filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file" + "github.com/grafana/grafana/pkg/services/apiserver/utils" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/featuremgmt" - entitystorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/entity" - filestorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/file" + "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/store/entity" "github.com/grafana/grafana/pkg/services/store/entity/db/dbimpl" "github.com/grafana/grafana/pkg/services/store/entity/sqlstash" "github.com/grafana/grafana/pkg/setting" ) -type StorageType string - -const ( - StorageTypeFile StorageType = "file" - StorageTypeEtcd StorageType = "etcd" - StorageTypeLegacy StorageType = "legacy" - StorageTypeUnified StorageType = "unified" - StorageTypeUnifiedGrpc StorageType = "unified-grpc" -) - var ( _ Service = (*service)(nil) _ RestConfigProvider = (*service)(nil) @@ -90,10 +73,6 @@ type Service interface { registry.CanBeDisabled } -type APIRegistrar interface { - RegisterAPI(builder APIGroupBuilder) -} - type RestConfigProvider interface { GetRestConfig() *clientrest.Config } @@ -111,7 +90,7 @@ type DirectRestConfigProvider interface { type service struct { *services.BasicService - config *config + options *grafanaapiserveroptions.Options restConfig *clientrest.Config cfg *setting.Cfg @@ -123,7 +102,7 @@ type service struct { db db.DB rr routing.RouteRegister handler http.Handler - builders []APIGroupBuilder + builders []builder.APIGroupBuilder tracing *tracing.TracingService @@ -139,12 +118,11 @@ func ProvideService( db db.DB, ) (*service, error) { s := &service{ - config: newConfig(cfg, features), cfg: cfg, features: features, rr: rr, stopCh: make(chan struct{}), - builders: []APIGroupBuilder{}, + builders: []builder.APIGroupBuilder{}, authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService), tracing: tracing, db: db, // For Unified storage @@ -182,7 +160,9 @@ func ProvideService( } s.rr.Group("/apis", proxyHandler) - s.rr.Group("/apiserver-metrics", proxyHandler) + s.rr.Group("/livez", proxyHandler) + s.rr.Group("/readyz", proxyHandler) + s.rr.Group("/healthz", proxyHandler) s.rr.Group("/openapi", proxyHandler) return s, nil @@ -193,7 +173,7 @@ func (s *service) GetRestConfig() *clientrest.Config { } func (s *service) IsDisabled() bool { - return !s.config.enabled + return false } // Run is an adapter for the BackgroundService interface. @@ -204,17 +184,11 @@ func (s *service) Run(ctx context.Context) error { return s.running(ctx) } -func (s *service) RegisterAPI(builder APIGroupBuilder) { - s.builders = append(s.builders, builder) +func (s *service) RegisterAPI(b builder.APIGroupBuilder) { + s.builders = append(s.builders, b) } func (s *service) start(ctx context.Context) error { - logger := logr.New(newLogAdapter(s.config.logLevel)) - klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true)) - if _, err := logs.GlogSetter(strconv.Itoa(s.config.logLevel)); err != nil { - logger.Error(err, "failed to set log level") - } - // Get the list of groups the server will support builders := s.builders @@ -226,59 +200,42 @@ func (s *service) start(ctx context.Context) error { return err } - // Optionally register a custom authorizer auth := b.GetAuthorizer() if auth != nil { s.authorizer.Register(b.GetGroupVersion(), auth) } } - o := options.NewRecommendedOptions("/registry/grafana.app", Codecs.LegacyCodec(groupVersions...)) - o.SecureServing.BindAddress = s.config.ip - o.SecureServing.BindPort = s.config.port - o.Authentication.RemoteKubeConfigFileOptional = true - o.Authorization.RemoteKubeConfigFileOptional = true + o := grafanaapiserveroptions.NewOptions(Codecs.LegacyCodec(groupVersions...)) + applyGrafanaConfig(s.cfg, s.features, o) - o.Admission = nil - o.CoreAPI = nil - - serverConfig := genericapiserver.NewRecommendedConfig(Codecs) - serverConfig.ExternalAddress = s.config.host - - if s.config.devMode { - // SecureServingOptions is used when the apiserver needs it's own listener. - // this is not needed in production, but it's useful for development kubectl access. - if err := o.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { - return err - } - // AuthenticationOptions is needed to authenticate requests from kubectl in dev mode. - if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil { - return err - } - } else { - // In production mode, override ExternalAddress and LoopbackClientConfig. - // In dev mode we want to use the loopback client config - // and address provided by SecureServingOptions. - serverConfig.ExternalAddress = s.config.host - serverConfig.LoopbackClientConfig = &clientrest.Config{ - Host: s.config.apiURL, - TLSClientConfig: clientrest.TLSClientConfig{ - Insecure: true, - }, - } + if errs := o.Validate(); len(errs) != 0 { + // TODO: handle multiple errors + return errs[0] } - switch s.config.storageType { - case StorageTypeEtcd: - o.Etcd.StorageConfig.Transport.ServerList = s.config.etcdServers - if err := o.Etcd.Validate(); len(err) > 0 { + serverConfig := genericapiserver.NewRecommendedConfig(Codecs) + if err := o.ApplyTo(serverConfig); err != nil { + return err + } + serverConfig.Authorization.Authorizer = s.authorizer + serverConfig.TracerProvider = s.tracing.GetTracerProvider() + + // setup loopback transport + transport := &roundTripperFunc{ready: make(chan struct{})} + serverConfig.LoopbackClientConfig.Transport = transport + serverConfig.LoopbackClientConfig.TLSClientConfig = clientrest.TLSClientConfig{} + + switch o.StorageOptions.StorageType { + case grafanaapiserveroptions.StorageTypeEtcd: + if err := o.RecommendedOptions.Etcd.Validate(); len(err) > 0 { return err[0] } - if err := o.Etcd.ApplyTo(&serverConfig.Config); err != nil { + if err := o.RecommendedOptions.Etcd.ApplyTo(&serverConfig.Config); err != nil { return err } - case StorageTypeUnified: + case grafanaapiserveroptions.StorageTypeUnified: if !s.features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorage) { return fmt.Errorf("unified storage requires the unifiedStorage feature flag (and app_mode = development)") } @@ -293,9 +250,9 @@ func (s *service) start(ctx context.Context) error { return err } - serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.Etcd.StorageConfig.Codec) + serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.RecommendedOptions.Etcd.StorageConfig.Codec) - case StorageTypeUnifiedGrpc: + case grafanaapiserveroptions.StorageTypeUnifiedGrpc: // Create a connection to the gRPC server // TODO: support configuring the gRPC server address conn, err := grpc.Dial("localhost:10000", grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -309,20 +266,16 @@ func (s *service) start(ctx context.Context) error { // Create a client instance store := entity.NewEntityStoreClientWrapper(conn) - serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.Etcd.StorageConfig.Codec) + serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.RecommendedOptions.Etcd.StorageConfig.Codec) - case StorageTypeFile: - serverConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(s.config.dataPath, o.Etcd.StorageConfig) - - case StorageTypeLegacy: - // do nothing? + case grafanaapiserveroptions.StorageTypeLegacy: + fallthrough + case grafanaapiserveroptions.StorageTypeFile: + serverConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(o.StorageOptions.DataPath, o.RecommendedOptions.Etcd.StorageConfig) } - serverConfig.Authorization.Authorizer = s.authorizer - serverConfig.TracerProvider = s.tracing.GetTracerProvider() - // Add OpenAPI specs for each group+version - err := SetupConfig(serverConfig, builders) + err := builder.SetupConfig(Scheme, serverConfig, builders) if err != nil { return err } @@ -339,28 +292,37 @@ func (s *service) start(ctx context.Context) error { return err } + dualWriteEnabled := o.StorageOptions.StorageType != grafanaapiserveroptions.StorageTypeLegacy + // Install the API Group+version - err = InstallAPIs(server, serverConfig.RESTOptionsGetter, builders) + err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, dualWriteEnabled) if err != nil { return err } + // set the transport function and signal that it's ready + transport.fn = func(req *http.Request) (*http.Response, error) { + w := newWrappedResponseWriter() + resp := responsewriter.WrapForHTTP1Or2(w) + server.Handler.ServeHTTP(resp, req) + return w.Result(), nil + } + close(transport.ready) + + // only write kubeconfig in dev mode + if o.ExtraOptions.DevMode { + if err := ensureKubeConfig(server.LoopbackClientConfig, o.StorageOptions.DataPath); err != nil { + return err + } + } + // Used by the proxy wrapper registered in ProvideService s.handler = server.Handler s.restConfig = server.LoopbackClientConfig + s.options = o prepared := server.PrepareRun() - // When running in production, do not start a standalone https server - if !s.config.devMode { - return nil - } - - // only write kubeconfig in dev mode - if err := s.ensureKubeConfig(); err != nil { - return err - } - go func() { s.stoppedCh <- prepared.Run(s.stopCh) }() @@ -385,12 +347,6 @@ func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) { } func (s *service) running(ctx context.Context) error { - // skip waiting for the server in prod mode - if !s.config.devMode { - <-ctx.Done() - return nil - } - select { case err := <-s.stoppedCh: if err != nil { @@ -402,47 +358,42 @@ func (s *service) running(ctx context.Context) error { return nil } -func (s *service) ensureKubeConfig() error { +func ensureKubeConfig(restConfig *clientrest.Config, dir string) error { return clientcmd.WriteToFile( - utils.FormatKubeConfig(s.restConfig), - path.Join(s.config.dataPath, "grafana.kubeconfig"), + utils.FormatKubeConfig(restConfig), + path.Join(dir, "grafana.kubeconfig"), ) } type roundTripperFunc struct { - fn func(req *http.Request) (*http.Response, error) + ready chan struct{} + fn func(req *http.Request) (*http.Response, error) } func (f *roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + if f.fn == nil { + <-f.ready + } return f.fn(req) } -// find the k8s version according to build info -func getK8sApiserverVersion() (string, error) { - bi, ok := debug.ReadBuildInfo() - if !ok { - return "", fmt.Errorf("debug.ReadBuildInfo() failed") - } +var _ http.ResponseWriter = (*wrappedResponseWriter)(nil) +var _ responsewriter.UserProvidedDecorator = (*wrappedResponseWriter)(nil) - if len(bi.Deps) == 0 { - return "v?.?", nil // this is normal while debugging - } +type wrappedResponseWriter struct { + *httptest.ResponseRecorder +} - for _, dep := range bi.Deps { - if dep.Path == "k8s.io/apiserver" { - if !semver.IsValid(dep.Version) { - return "", fmt.Errorf("invalid semantic version for k8s.io/apiserver") - } - // v0 => v1 - majorVersion := strings.TrimPrefix(semver.Major(dep.Version), "v") - majorInt, err := strconv.Atoi(majorVersion) - if err != nil { - return "", fmt.Errorf("could not convert majorVersion to int. majorVersion: %s", majorVersion) - } - newMajor := fmt.Sprintf("v%d", majorInt+1) - return strings.Replace(dep.Version, semver.Major(dep.Version), newMajor, 1), nil - } - } +func newWrappedResponseWriter() *wrappedResponseWriter { + w := httptest.NewRecorder() + return &wrappedResponseWriter{w} +} - return "", fmt.Errorf("could not find k8s.io/apiserver in build info") +func (w *wrappedResponseWriter) Unwrap() http.ResponseWriter { + return w.ResponseRecorder +} + +func (w *wrappedResponseWriter) CloseNotify() <-chan bool { + // TODO: this is probably not the right thing to do here + return make(<-chan bool) } diff --git a/pkg/services/grafana-apiserver/storage/entity/fieldRequirements.go b/pkg/services/apiserver/storage/entity/fieldRequirements.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/fieldRequirements.go rename to pkg/services/apiserver/storage/entity/fieldRequirements.go diff --git a/pkg/services/grafana-apiserver/storage/entity/restoptions.go b/pkg/services/apiserver/storage/entity/restoptions.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/restoptions.go rename to pkg/services/apiserver/storage/entity/restoptions.go diff --git a/pkg/services/grafana-apiserver/storage/entity/storage.go b/pkg/services/apiserver/storage/entity/storage.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/storage.go rename to pkg/services/apiserver/storage/entity/storage.go diff --git a/pkg/services/grafana-apiserver/storage/entity/utils.go b/pkg/services/apiserver/storage/entity/utils.go similarity index 98% rename from pkg/services/grafana-apiserver/storage/entity/utils.go rename to pkg/services/apiserver/storage/entity/utils.go index b20e40c9ebe..22b122e43e6 100644 --- a/pkg/services/grafana-apiserver/storage/entity/utils.go +++ b/pkg/services/apiserver/storage/entity/utils.go @@ -15,7 +15,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/endpoints/request" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/utils" entityStore "github.com/grafana/grafana/pkg/services/store/entity" ) diff --git a/pkg/services/grafana-apiserver/storage/entity/utils_test.go b/pkg/services/apiserver/storage/entity/utils_test.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/entity/utils_test.go rename to pkg/services/apiserver/storage/entity/utils_test.go diff --git a/pkg/services/grafana-apiserver/storage/file/file.go b/pkg/services/apiserver/storage/file/file.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/file.go rename to pkg/services/apiserver/storage/file/file.go diff --git a/pkg/services/grafana-apiserver/storage/file/restoptions.go b/pkg/services/apiserver/storage/file/restoptions.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/restoptions.go rename to pkg/services/apiserver/storage/file/restoptions.go diff --git a/pkg/services/grafana-apiserver/storage/file/util.go b/pkg/services/apiserver/storage/file/util.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/util.go rename to pkg/services/apiserver/storage/file/util.go diff --git a/pkg/services/grafana-apiserver/storage/file/watchset.go b/pkg/services/apiserver/storage/file/watchset.go similarity index 100% rename from pkg/services/grafana-apiserver/storage/file/watchset.go rename to pkg/services/apiserver/storage/file/watchset.go diff --git a/pkg/services/grafana-apiserver/utils/clientConfig.go b/pkg/services/apiserver/utils/clientConfig.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/clientConfig.go rename to pkg/services/apiserver/utils/clientConfig.go diff --git a/pkg/services/grafana-apiserver/utils/meta.go b/pkg/services/apiserver/utils/meta.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/meta.go rename to pkg/services/apiserver/utils/meta.go diff --git a/pkg/services/grafana-apiserver/utils/meta_test.go b/pkg/services/apiserver/utils/meta_test.go similarity index 98% rename from pkg/services/grafana-apiserver/utils/meta_test.go rename to pkg/services/apiserver/utils/meta_test.go index 9d9cfedd3ce..e8555bdff6e 100644 --- a/pkg/services/grafana-apiserver/utils/meta_test.go +++ b/pkg/services/apiserver/utils/meta_test.go @@ -8,7 +8,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/utils" ) type TestResource struct { diff --git a/pkg/services/grafana-apiserver/utils/tableConverter.go b/pkg/services/apiserver/utils/tableConverter.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/tableConverter.go rename to pkg/services/apiserver/utils/tableConverter.go diff --git a/pkg/services/grafana-apiserver/utils/tableConverter_test.go b/pkg/services/apiserver/utils/tableConverter_test.go similarity index 98% rename from pkg/services/grafana-apiserver/utils/tableConverter_test.go rename to pkg/services/apiserver/utils/tableConverter_test.go index ec1310475c9..7c7618d4f0d 100644 --- a/pkg/services/grafana-apiserver/utils/tableConverter_test.go +++ b/pkg/services/apiserver/utils/tableConverter_test.go @@ -11,7 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/utils" + "github.com/grafana/grafana/pkg/services/apiserver/utils" ) func TestTableConverter(t *testing.T) { diff --git a/pkg/services/grafana-apiserver/utils/uids.go b/pkg/services/apiserver/utils/uids.go similarity index 100% rename from pkg/services/grafana-apiserver/utils/uids.go rename to pkg/services/apiserver/utils/uids.go diff --git a/pkg/services/grafana-apiserver/wireset.go b/pkg/services/apiserver/wireset.go similarity index 63% rename from pkg/services/grafana-apiserver/wireset.go rename to pkg/services/apiserver/wireset.go index 25a02e898ef..f7163f1ad6f 100644 --- a/pkg/services/grafana-apiserver/wireset.go +++ b/pkg/services/apiserver/wireset.go @@ -1,13 +1,15 @@ -package grafanaapiserver +package apiserver import ( "github.com/google/wire" + + "github.com/grafana/grafana/pkg/services/apiserver/builder" ) var WireSet = wire.NewSet( ProvideService, wire.Bind(new(RestConfigProvider), new(*service)), wire.Bind(new(Service), new(*service)), - wire.Bind(new(APIRegistrar), new(*service)), wire.Bind(new(DirectRestConfigProvider), new(*service)), + wire.Bind(new(builder.APIRegistrar), new(*service)), ) diff --git a/pkg/services/grafana-apiserver/config.go b/pkg/services/grafana-apiserver/config.go deleted file mode 100644 index df30eb12eb9..00000000000 --- a/pkg/services/grafana-apiserver/config.go +++ /dev/null @@ -1,60 +0,0 @@ -package grafanaapiserver - -import ( - "fmt" - "net" - "path/filepath" - "strconv" - - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" -) - -type config struct { - enabled bool - devMode bool - - ip net.IP - port int - host string - apiURL string - - storageType StorageType - - etcdServers []string - dataPath string - - logLevel int -} - -func newConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles) *config { - defaultLogLevel := 0 - ip := net.ParseIP(cfg.HTTPAddr) - apiURL := cfg.AppURL - port, err := strconv.Atoi(cfg.HTTPPort) - if err != nil { - port = 3000 - } - - if cfg.Env == setting.Dev { - defaultLogLevel = 10 - port = 6443 - ip = net.ParseIP("127.0.0.1") - apiURL = fmt.Sprintf("https://%s:%d", ip, port) - } - - host := fmt.Sprintf("%s:%d", ip, port) - - return &config{ - enabled: true, - devMode: features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess), - dataPath: filepath.Join(cfg.DataPath, "grafana-apiserver"), - ip: ip, - port: port, - host: host, - logLevel: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("log_level").MustInt(defaultLogLevel), - etcdServers: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(","), - storageType: StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(StorageTypeLegacy))), - apiURL: apiURL, - } -} diff --git a/pkg/services/grafana-apiserver/config_test.go b/pkg/services/grafana-apiserver/config_test.go deleted file mode 100644 index e76fe5e7f93..00000000000 --- a/pkg/services/grafana-apiserver/config_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package grafanaapiserver - -import ( - "net" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" -) - -func TestNewConfig(t *testing.T) { - cfg := setting.NewCfg() - cfg.Env = setting.Prod - cfg.DataPath = "/tmp/grafana" - cfg.HTTPAddr = "10.0.0.1" - cfg.HTTPPort = "4000" - cfg.AppURL = "http://test:4000" - - section := cfg.Raw.Section("grafana-apiserver") - section.Key("log_level").SetValue("5") - section.Key("etcd_servers").SetValue("http://localhost:2379") - - actual := newConfig(cfg, featuremgmt.WithFeatures()) - - expected := &config{ - enabled: true, - devMode: false, - storageType: StorageTypeLegacy, - etcdServers: []string{"http://localhost:2379"}, - apiURL: "http://test:4000", - ip: net.ParseIP("10.0.0.1"), - port: 4000, - host: "10.0.0.1:4000", - dataPath: "/tmp/grafana/grafana-apiserver", - logLevel: 5, - } - require.Equal(t, expected, actual) -} diff --git a/pkg/tests/apis/helper.go b/pkg/tests/apis/helper.go index 3d1e233174d..8faebfdfda7 100644 --- a/pkg/tests/apis/helper.go +++ b/pkg/tests/apis/helper.go @@ -25,9 +25,9 @@ import ( "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/server" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/auth/identity" "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/org/orgimpl" "github.com/grafana/grafana/pkg/services/quota/quotaimpl" From f73d0eb41c031425a0069d49fd73203a2b753401 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:08:02 -0600 Subject: [PATCH 75/84] Transformations: Expose "keep fields" option in partition by values (#81743) * expose keep fields to partitionByValues UI --- .../partitionByValues/PartitionByValuesEditor.tsx | 14 ++++++++++++++ .../partitionByValues/partitionByValues.ts | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx b/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx index 874516506e0..167ba484b6e 100644 --- a/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx +++ b/public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx @@ -67,6 +67,11 @@ export function PartitionByValuesEditor({ { label: 'As frame name', value: namingModes.frameName }, ]; + const KeepFieldsOptions = [ + { label: 'Yes', value: true }, + { label: 'No', value: false }, + ]; + const removeField = useCallback( (v: string) => { if (!v) { @@ -135,6 +140,15 @@ export function PartitionByValuesEditor({ /> + + + onChange({ ...options, keepFields: v })} + /> + +

); } diff --git a/public/app/features/transformers/partitionByValues/partitionByValues.ts b/public/app/features/transformers/partitionByValues/partitionByValues.ts index d33d00f04c7..8368d24806d 100644 --- a/public/app/features/transformers/partitionByValues/partitionByValues.ts +++ b/public/app/features/transformers/partitionByValues/partitionByValues.ts @@ -67,7 +67,9 @@ export const partitionByValuesTransformer: SynchronousDataTransformerInfo (source) => source.pipe(map((data) => partitionByValuesTransformer.transformer(options, ctx)(data))), From 147db1a667bc044ac64235547183f0314d2da087 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:25:31 +0000 Subject: [PATCH 76/84] Update dependency @testing-library/jest-dom to v6.4.1 --- package.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-ui/package.json | 2 +- .../grafana-pyroscope-datasource/package.json | 2 +- .../app/plugins/datasource/tempo/package.json | 2 +- yarn.lock | 18 +++++++++--------- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index fb28910137d..136ee52c44d 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,7 @@ "@swc/core": "1.3.107", "@swc/helpers": "0.5.3", "@testing-library/dom": "9.3.4", - "@testing-library/jest-dom": "6.4.0", + "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", "@types/angular": "1.8.9", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 54922002175..445c2074244 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -67,7 +67,7 @@ "@rollup/plugin-json": "6.1.0", "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/dom": "9.3.4", - "@testing-library/jest-dom": "6.4.0", + "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", "@types/dompurify": "^3.0.0", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index b37a30177cd..81ce88168ff 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -129,7 +129,7 @@ "@storybook/react-webpack5": "7.4.5", "@storybook/theming": "7.4.5", "@testing-library/dom": "9.3.4", - "@testing-library/jest-dom": "6.4.0", + "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", "@types/common-tags": "^1.8.0", diff --git a/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json b/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json index 5b047e14cbb..8fa032e554a 100644 --- a/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json +++ b/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json @@ -21,7 +21,7 @@ }, "devDependencies": { "@grafana/plugin-configs": "10.4.0-pre", - "@testing-library/jest-dom": "6.4.0", + "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", "@types/jest": "29.5.11", diff --git a/public/app/plugins/datasource/tempo/package.json b/public/app/plugins/datasource/tempo/package.json index 72bb414a789..50e274c8268 100644 --- a/public/app/plugins/datasource/tempo/package.json +++ b/public/app/plugins/datasource/tempo/package.json @@ -42,7 +42,7 @@ }, "devDependencies": { "@grafana/plugin-configs": "10.4.0-pre", - "@testing-library/jest-dom": "6.4.0", + "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.0", "@testing-library/user-event": "14.5.2", "@types/jest": "29.5.11", diff --git a/yarn.lock b/yarn.lock index c3a56996a52..098e1acceb4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3119,7 +3119,7 @@ __metadata: "@grafana/runtime": "npm:10.4.0-pre" "@grafana/schema": "npm:10.4.0-pre" "@grafana/ui": "npm:10.4.0-pre" - "@testing-library/jest-dom": "npm:6.4.0" + "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.0" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:29.5.11" @@ -3256,7 +3256,7 @@ __metadata: "@opentelemetry/exporter-collector": "npm:0.25.0" "@opentelemetry/semantic-conventions": "npm:1.21.0" "@reduxjs/toolkit": "npm:1.9.5" - "@testing-library/jest-dom": "npm:6.4.0" + "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.0" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:29.5.11" @@ -3325,7 +3325,7 @@ __metadata: "@rollup/plugin-json": "npm:6.1.0" "@rollup/plugin-node-resolve": "npm:15.2.3" "@testing-library/dom": "npm:9.3.4" - "@testing-library/jest-dom": "npm:6.4.0" + "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.0" "@testing-library/user-event": "npm:14.5.2" "@types/d3-interpolate": "npm:^3.0.0" @@ -3860,7 +3860,7 @@ __metadata: "@storybook/react-webpack5": "npm:7.4.5" "@storybook/theming": "npm:7.4.5" "@testing-library/dom": "npm:9.3.4" - "@testing-library/jest-dom": "npm:6.4.0" + "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.0" "@testing-library/user-event": "npm:14.5.2" "@types/common-tags": "npm:^1.8.0" @@ -8007,9 +8007,9 @@ __metadata: languageName: node linkType: hard -"@testing-library/jest-dom@npm:6.4.0, @testing-library/jest-dom@npm:^6.1.2": - version: 6.4.0 - resolution: "@testing-library/jest-dom@npm:6.4.0" +"@testing-library/jest-dom@npm:6.4.1, @testing-library/jest-dom@npm:^6.1.2": + version: 6.4.1 + resolution: "@testing-library/jest-dom@npm:6.4.1" dependencies: "@adobe/css-tools": "npm:^4.3.2" "@babel/runtime": "npm:^7.9.2" @@ -8036,7 +8036,7 @@ __metadata: optional: true vitest: optional: true - checksum: 10/09179611c61357925ca26a9cde4398a6ceee87940ab1c991c61d76fa74adb80e637f60e2ad1443e2cc75f1d336dc6285d5f50419e37b9bd04ad585233f913fc8 + checksum: 10/ae042927361fb0c1bfe9fb5ed7cbcb7d5acff3236bfad203e4cd99d83e4b6047624df3b9c5fa3199534e6885e4ac5d3777a2a5f536f3b65390218ac48fae50a1 languageName: node linkType: hard @@ -17659,7 +17659,7 @@ __metadata: "@swc/core": "npm:1.3.107" "@swc/helpers": "npm:0.5.3" "@testing-library/dom": "npm:9.3.4" - "@testing-library/jest-dom": "npm:6.4.0" + "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.0" "@testing-library/react-hooks": "npm:^8.0.1" "@testing-library/user-event": "npm:14.5.2" From e6a4992f37116092a60eef16fb964566bf211fc1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 23:16:08 +0000 Subject: [PATCH 77/84] Update dependency @testing-library/react to v14.2.1 --- package.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-flamegraph/package.json | 2 +- .../grafana-o11y-ds-frontend/package.json | 2 +- packages/grafana-runtime/package.json | 2 +- packages/grafana-sql/package.json | 2 +- packages/grafana-ui/package.json | 2 +- .../datasource/azuremonitor/package.json | 2 +- .../datasource/cloud-monitoring/package.json | 2 +- .../grafana-pyroscope-datasource/package.json | 2 +- .../grafana-testdata-datasource/package.json | 2 +- .../app/plugins/datasource/parca/package.json | 2 +- .../app/plugins/datasource/tempo/package.json | 2 +- yarn.lock | 34 +++++++++---------- 14 files changed, 30 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index 136ee52c44d..7966e727bc0 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "@swc/helpers": "0.5.3", "@testing-library/dom": "9.3.4", "@testing-library/jest-dom": "6.4.1", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/angular": "1.8.9", "@types/angular-route": "1.7.6", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 445c2074244..6db7dec4a71 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -68,7 +68,7 @@ "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/dom": "9.3.4", "@testing-library/jest-dom": "6.4.1", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/dompurify": "^3.0.0", "@types/history": "4.7.11", diff --git a/packages/grafana-flamegraph/package.json b/packages/grafana-flamegraph/package.json index 6daf416e156..66f3ee1132c 100644 --- a/packages/grafana-flamegraph/package.json +++ b/packages/grafana-flamegraph/package.json @@ -62,7 +62,7 @@ "@grafana/tsconfig": "^1.2.0-rc1", "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/jest-dom": "^6.1.2", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/d3": "^7", "@types/jest": "^29.5.4", diff --git a/packages/grafana-o11y-ds-frontend/package.json b/packages/grafana-o11y-ds-frontend/package.json index db3f6df8f4a..213dcb8c6a2 100644 --- a/packages/grafana-o11y-ds-frontend/package.json +++ b/packages/grafana-o11y-ds-frontend/package.json @@ -32,7 +32,7 @@ "devDependencies": { "@grafana/tsconfig": "^1.2.0-rc1", "@testing-library/jest-dom": "^6.1.2", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/jest": "^29.5.4", "@types/react": "18.2.48", diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index a724a669be9..d04ed92bcf0 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -54,7 +54,7 @@ "@rollup/plugin-commonjs": "25.0.7", "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/dom": "9.3.4", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/angular": "1.8.9", "@types/history": "4.7.11", diff --git a/packages/grafana-sql/package.json b/packages/grafana-sql/package.json index c5e632a078f..118057db472 100644 --- a/packages/grafana-sql/package.json +++ b/packages/grafana-sql/package.json @@ -35,7 +35,7 @@ "devDependencies": { "@grafana/tsconfig": "^1.3.0-rc1", "@testing-library/jest-dom": "^6.1.2", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/react-hooks": "^8.0.1", "@testing-library/user-event": "14.5.2", "@types/jest": "^29.5.4", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 81ce88168ff..cb9dcc16f38 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -130,7 +130,7 @@ "@storybook/theming": "7.4.5", "@testing-library/dom": "9.3.4", "@testing-library/jest-dom": "6.4.1", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/common-tags": "^1.8.0", "@types/d3": "7.4.3", diff --git a/public/app/plugins/datasource/azuremonitor/package.json b/public/app/plugins/datasource/azuremonitor/package.json index 740b0339a58..68dda36c97f 100644 --- a/public/app/plugins/datasource/azuremonitor/package.json +++ b/public/app/plugins/datasource/azuremonitor/package.json @@ -25,7 +25,7 @@ "devDependencies": { "@grafana/e2e-selectors": "10.4.0-pre", "@grafana/plugin-configs": "10.4.0-pre", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/jest": "29.5.11", "@types/lodash": "4.14.202", diff --git a/public/app/plugins/datasource/cloud-monitoring/package.json b/public/app/plugins/datasource/cloud-monitoring/package.json index e64cea2a291..e849f3ee6b5 100644 --- a/public/app/plugins/datasource/cloud-monitoring/package.json +++ b/public/app/plugins/datasource/cloud-monitoring/package.json @@ -27,7 +27,7 @@ "devDependencies": { "@grafana/e2e-selectors": "10.4.0-pre", "@grafana/plugin-configs": "10.4.0-pre", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/debounce-promise": "3.1.9", "@types/jest": "29.5.11", diff --git a/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json b/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json index 8fa032e554a..0ce6d3febc6 100644 --- a/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json +++ b/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json @@ -22,7 +22,7 @@ "devDependencies": { "@grafana/plugin-configs": "10.4.0-pre", "@testing-library/jest-dom": "6.4.1", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/jest": "29.5.11", "@types/lodash": "4.14.202", diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/package.json b/public/app/plugins/datasource/grafana-testdata-datasource/package.json index 547ed50b2db..c1301cbd0ee 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/package.json +++ b/public/app/plugins/datasource/grafana-testdata-datasource/package.json @@ -22,7 +22,7 @@ "devDependencies": { "@grafana/e2e-selectors": "10.4.0-pre", "@grafana/plugin-configs": "10.4.0-pre", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/d3-random": "^3.0.2", "@types/jest": "29.5.11", diff --git a/public/app/plugins/datasource/parca/package.json b/public/app/plugins/datasource/parca/package.json index 7adea4c992e..5e11749a5cc 100644 --- a/public/app/plugins/datasource/parca/package.json +++ b/public/app/plugins/datasource/parca/package.json @@ -18,7 +18,7 @@ }, "devDependencies": { "@grafana/plugin-configs": "10.4.0-pre", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/lodash": "4.14.202", "@types/react": "18.2.48", diff --git a/public/app/plugins/datasource/tempo/package.json b/public/app/plugins/datasource/tempo/package.json index 50e274c8268..3d9abdfb7b8 100644 --- a/public/app/plugins/datasource/tempo/package.json +++ b/public/app/plugins/datasource/tempo/package.json @@ -43,7 +43,7 @@ "devDependencies": { "@grafana/plugin-configs": "10.4.0-pre", "@testing-library/jest-dom": "6.4.1", - "@testing-library/react": "14.2.0", + "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/jest": "29.5.11", "@types/lodash": "4.14.202", diff --git a/yarn.lock b/yarn.lock index 098e1acceb4..f8980dc2278 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3037,7 +3037,7 @@ __metadata: "@grafana/schema": "npm:10.4.0-pre" "@grafana/ui": "npm:10.4.0-pre" "@kusto/monaco-kusto": "npm:^7.4.0" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:29.5.11" "@types/lodash": "npm:4.14.202" @@ -3078,7 +3078,7 @@ __metadata: "@grafana/schema": "npm:10.4.0-pre" "@grafana/ui": "npm:10.4.0-pre" "@kusto/monaco-kusto": "npm:^7.4.0" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/debounce-promise": "npm:3.1.9" "@types/jest": "npm:29.5.11" @@ -3120,7 +3120,7 @@ __metadata: "@grafana/schema": "npm:10.4.0-pre" "@grafana/ui": "npm:10.4.0-pre" "@testing-library/jest-dom": "npm:6.4.1" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:29.5.11" "@types/lodash": "npm:4.14.202" @@ -3160,7 +3160,7 @@ __metadata: "@grafana/runtime": "npm:10.4.0-pre" "@grafana/schema": "npm:10.4.0-pre" "@grafana/ui": "npm:10.4.0-pre" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/d3-random": "npm:^3.0.2" "@types/jest": "npm:29.5.11" @@ -3217,7 +3217,7 @@ __metadata: "@grafana/runtime": "npm:10.4.0-pre" "@grafana/schema": "npm:10.4.0-pre" "@grafana/ui": "npm:10.4.0-pre" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/lodash": "npm:4.14.202" "@types/react": "npm:18.2.48" @@ -3257,7 +3257,7 @@ __metadata: "@opentelemetry/semantic-conventions": "npm:1.21.0" "@reduxjs/toolkit": "npm:1.9.5" "@testing-library/jest-dom": "npm:6.4.1" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:29.5.11" "@types/lodash": "npm:4.14.202" @@ -3326,7 +3326,7 @@ __metadata: "@rollup/plugin-node-resolve": "npm:15.2.3" "@testing-library/dom": "npm:9.3.4" "@testing-library/jest-dom": "npm:6.4.1" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/d3-interpolate": "npm:^3.0.0" "@types/dompurify": "npm:^3.0.0" @@ -3567,7 +3567,7 @@ __metadata: "@leeoniya/ufuzzy": "npm:1.0.14" "@rollup/plugin-node-resolve": "npm:15.2.3" "@testing-library/jest-dom": "npm:^6.1.2" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/d3": "npm:^7" "@types/jest": "npm:^29.5.4" @@ -3650,7 +3650,7 @@ __metadata: "@grafana/tsconfig": "npm:^1.2.0-rc1" "@grafana/ui": "workspace:*" "@testing-library/jest-dom": "npm:^6.1.2" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:^29.5.4" "@types/react": "npm:18.2.48" @@ -3699,7 +3699,7 @@ __metadata: "@rollup/plugin-commonjs": "npm:25.0.7" "@rollup/plugin-node-resolve": "npm:15.2.3" "@testing-library/dom": "npm:9.3.4" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/angular": "npm:1.8.9" "@types/history": "npm:4.7.11" @@ -3781,7 +3781,7 @@ __metadata: "@grafana/ui": "workspace:*" "@react-awesome-query-builder/ui": "npm:6.4.2" "@testing-library/jest-dom": "npm:^6.1.2" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/react-hooks": "npm:^8.0.1" "@testing-library/user-event": "npm:14.5.2" "@types/jest": "npm:^29.5.4" @@ -3861,7 +3861,7 @@ __metadata: "@storybook/theming": "npm:7.4.5" "@testing-library/dom": "npm:9.3.4" "@testing-library/jest-dom": "npm:6.4.1" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/common-tags": "npm:^1.8.0" "@types/d3": "npm:7.4.3" @@ -8062,9 +8062,9 @@ __metadata: languageName: node linkType: hard -"@testing-library/react@npm:14.2.0": - version: 14.2.0 - resolution: "@testing-library/react@npm:14.2.0" +"@testing-library/react@npm:14.2.1": + version: 14.2.1 + resolution: "@testing-library/react@npm:14.2.1" dependencies: "@babel/runtime": "npm:^7.12.5" "@testing-library/dom": "npm:^9.0.0" @@ -8072,7 +8072,7 @@ __metadata: peerDependencies: react: ^18.0.0 react-dom: ^18.0.0 - checksum: 10/e32b6958141215b8a0c0637df8f27efb01da8e7c311f9faedb32bb08ce976632a31b10df4e6d4c02d63e5bdf1974b0d04f3f375be0f26a86727cdd73619fffc2 + checksum: 10/e02b2f32ae79665a79fc4d8ee053fd3832bfcd4753aa1dba05cdece1a9f59c72a0fae91e0a9387597dcb686d631a722729f2878e38dc95e6f23b291ad8d09b6c languageName: node linkType: hard @@ -17660,7 +17660,7 @@ __metadata: "@swc/helpers": "npm:0.5.3" "@testing-library/dom": "npm:9.3.4" "@testing-library/jest-dom": "npm:6.4.1" - "@testing-library/react": "npm:14.2.0" + "@testing-library/react": "npm:14.2.1" "@testing-library/react-hooks": "npm:^8.0.1" "@testing-library/user-event": "npm:14.5.2" "@types/angular": "npm:1.8.9" From 10bde9ce9c2dbf2f31c8ddfaea25240788d7156a Mon Sep 17 00:00:00 2001 From: Isabel <76437239+imatwawana@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:25:54 -0500 Subject: [PATCH 78/84] Docs: add missing supported visualizations (#81668) * Added Supported visualizations sections and missing settings for value mappings * Updated supported visualizations table and added docs ref links * Updated supported visualizations table * Removed placeholder headings and accidentally added section to data links page * Added missing visualization --- .../configure-data-links/index.md | 59 +++++++++++++++++++ .../configure-value-mappings/index.md | 57 ++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/docs/sources/panels-visualizations/configure-data-links/index.md b/docs/sources/panels-visualizations/configure-data-links/index.md index d265641034c..b2fda39c1ac 100644 --- a/docs/sources/panels-visualizations/configure-data-links/index.md +++ b/docs/sources/panels-visualizations/configure-data-links/index.md @@ -38,6 +38,20 @@ For visualizations like stat, gauge, or bar gauge you can click anywhere on the If there's only one data link in the visualization, clicking anywhere on the visualization opens the link rather than the context menu. +## Supported visualizations + +You can configure data links for the following visualizations: + +| | | | +| -------------------------- | ---------------------- | -------------------------------- | +| [Bar chart][bar chart] | [Geomap][geomap] | [State timeline][state timeline] | +| [Bar gauge][bar gauge] | [Heatmap][heatmap] | [Status history][status history] | +| [Candlestick][candlestick] | [Histogram][histogram] | [Table][table] | +| [Canvas][canvas] | [Pie chart][pie chart] | [Time series][time series] | +| [Gauge][gauge] | [Stat][stat] | [Trend][trend] | + + + ## Data link variables Variables in data links let you send people to a detailed dashboard with preserved data filters. For example, you could use variables to specify a label, time range, series, or variable selection. @@ -136,6 +150,51 @@ If you want to add all of the current dashboard's variables to the URL, then use 1. Click the **Save dashboard** icon to save your changes to the dashboard. {{% docs/reference %}} +[bar chart]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/bar-chart" +[bar chart]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/bar-chart" + +[bar gauge]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/bar-gauge" +[bar gauge]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/bar-gauge" + +[candlestick]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/candlestick" +[candlestick]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/candlestick" + +[canvas]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/canvas" +[canvas]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/canvas" + +[gauge]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/gauge" +[gauge]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/gauge" + +[geomap]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/geomap" +[geomap]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/geomap" + +[heatmap]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/heatmap" +[heatmap]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/heatmap" + +[histogram]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/histogram" +[histogram]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/histogram" + +[pie chart]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/pie-chart" +[pie chart]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/pie-chart" + +[stat]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/stat" +[stat]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/stat" + +[state timeline]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/state-timeline" +[state timeline]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/state-timeline" + +[status history]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/status-history" +[status history]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/status-history" + +[table]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/table" +[table]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/table" + +[time series]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/time-series" +[time series]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/time-series" + +[trend]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/trend" +[trend]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/trend" + [Cloudwatch]: "/docs/grafana/ -> /docs/grafana//datasources/aws-cloudwatch/query-editor#deep-link-grafana-panels-to-the-cloudwatch-console-1" [Cloudwatch]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/connect-externally-hosted/data-sources/aws-cloudwatch/query-editor#deep-link-grafana-panels-to-the-cloudwatch-console-1" diff --git a/docs/sources/panels-visualizations/configure-value-mappings/index.md b/docs/sources/panels-visualizations/configure-value-mappings/index.md index 4a62ace44ec..be516356ec5 100644 --- a/docs/sources/panels-visualizations/configure-value-mappings/index.md +++ b/docs/sources/panels-visualizations/configure-value-mappings/index.md @@ -30,6 +30,18 @@ For example, the mapping applied in the following image causes the visualization Value mappings bypass unit formatting set in the **Standard options** section of panel editor, like color or number of decimal places displayed. When value mappings are present in a panel, Grafana displays a summary of them in the **Value mappings** section of the editor panel. +## Supported visualizations + +You can configure value mappings for the following visualizations: + +| | | | +| -------------------------- | -------------------------------- | -------------------------------- | +| [Bar chart][bar chart] | [Geomap][geomap] | [Status history][status history] | +| [Bar gauge][bar gauge] | [Histogram][histogram] | [Table][table] | +| [Candlestick][candlestick] | [Pie chart][pie chart] | [Time series][time series] | +| [Canvas][canvas] | [Stat][stat] | [Trend][trend] | +| [Gauge][gauge] | [State timeline][state timeline] | | + ## Types of value mappings Grafana supports the following value mapping types: @@ -98,6 +110,51 @@ The following image shows a table visualization with value mappings. If you want 1. (Optional) Enter display text. 1. (Optional) Set the color. +1. (Optional) Set an icon (canvas visualizations only). 1. Click **Update** to save the value mapping. After you've added a mapping, the **Edit value mappings** button replaces the **Add value mappings** button. Click the edit button to add or update mappings. + +{{% docs/reference %}} +[bar chart]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/bar-chart" +[bar chart]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/bar-chart" + +[bar gauge]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/bar-gauge" +[bar gauge]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/bar-gauge" + +[candlestick]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/candlestick" +[candlestick]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/candlestick" + +[canvas]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/canvas" +[canvas]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/canvas" + +[gauge]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/gauge" +[gauge]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/gauge" + +[geomap]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/geomap" +[geomap]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/geomap" + +[histogram]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/histogram" +[histogram]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/histogram" + +[pie chart]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/pie-chart" +[pie chart]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/pie-chart" + +[stat]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/stat" +[stat]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/stat" + +[state timeline]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/state-timeline" +[state timeline]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/state-timeline" + +[status history]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/status-history" +[status history]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/status-history" + +[table]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/table" +[table]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/table" + +[time series]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/time-series" +[time series]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/time-series" + +[trend]: "/docs/grafana/ -> /docs/grafana//panels-visualizations/visualizations/trend" +[trend]: "/docs/grafana-cloud/ -> /docs/grafana-cloud/visualizations/panels-visualizations/visualizations/trend" +{{% /docs/reference %}} From 315d3a7a72511377a053c0f4c0a84ff29f1b1447 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 23:43:21 +0000 Subject: [PATCH 79/84] Update dependency @types/jest to v29.5.12 --- package.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-runtime/package.json | 2 +- packages/grafana-ui/package.json | 2 +- .../datasource/azuremonitor/package.json | 2 +- .../datasource/cloud-monitoring/package.json | 2 +- .../grafana-pyroscope-datasource/package.json | 2 +- .../grafana-testdata-datasource/package.json | 2 +- .../app/plugins/datasource/tempo/package.json | 2 +- yarn.lock | 26 +++++++++---------- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 7966e727bc0..598c446d0c2 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "@types/gtag.js": "^0.0.18", "@types/history": "4.7.11", "@types/hoist-non-react-statics": "3.3.5", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/jquery": "3.5.29", "@types/js-yaml": "^4.0.5", "@types/jsurl": "^1.2.28", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 6db7dec4a71..87eaec0e421 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -72,7 +72,7 @@ "@testing-library/user-event": "14.5.2", "@types/dompurify": "^3.0.0", "@types/history": "4.7.11", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/jquery": "3.5.29", "@types/lodash": "4.14.202", "@types/marked": "5.0.2", diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index d04ed92bcf0..e746d83be55 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -58,7 +58,7 @@ "@testing-library/user-event": "14.5.2", "@types/angular": "1.8.9", "@types/history": "4.7.11", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/lodash": "4.14.202", "@types/react": "18.2.48", "@types/react-dom": "18.2.18", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index cb9dcc16f38..4092f042bf7 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -136,7 +136,7 @@ "@types/d3": "7.4.3", "@types/hoist-non-react-statics": "3.3.5", "@types/is-hotkey": "0.1.10", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/jquery": "3.5.29", "@types/lodash": "4.14.202", "@types/mock-raf": "1.0.6", diff --git a/public/app/plugins/datasource/azuremonitor/package.json b/public/app/plugins/datasource/azuremonitor/package.json index 68dda36c97f..aacbade2f61 100644 --- a/public/app/plugins/datasource/azuremonitor/package.json +++ b/public/app/plugins/datasource/azuremonitor/package.json @@ -27,7 +27,7 @@ "@grafana/plugin-configs": "10.4.0-pre", "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/lodash": "4.14.202", "@types/node": "20.11.13", "@types/prismjs": "1.26.3", diff --git a/public/app/plugins/datasource/cloud-monitoring/package.json b/public/app/plugins/datasource/cloud-monitoring/package.json index e849f3ee6b5..cd6cc00cf90 100644 --- a/public/app/plugins/datasource/cloud-monitoring/package.json +++ b/public/app/plugins/datasource/cloud-monitoring/package.json @@ -30,7 +30,7 @@ "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/debounce-promise": "3.1.9", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/lodash": "4.14.202", "@types/node": "20.11.13", "@types/prismjs": "1.26.3", diff --git a/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json b/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json index 0ce6d3febc6..939d0f53f45 100644 --- a/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json +++ b/public/app/plugins/datasource/grafana-pyroscope-datasource/package.json @@ -24,7 +24,7 @@ "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/lodash": "4.14.202", "@types/prismjs": "1.26.3", "@types/react": "18.2.48", diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/package.json b/public/app/plugins/datasource/grafana-testdata-datasource/package.json index c1301cbd0ee..a2e99cce0ff 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/package.json +++ b/public/app/plugins/datasource/grafana-testdata-datasource/package.json @@ -25,7 +25,7 @@ "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", "@types/d3-random": "^3.0.2", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/lodash": "4.14.202", "@types/node": "20.11.13", "@types/react": "18.2.48", diff --git a/public/app/plugins/datasource/tempo/package.json b/public/app/plugins/datasource/tempo/package.json index 3d9abdfb7b8..1a488957a21 100644 --- a/public/app/plugins/datasource/tempo/package.json +++ b/public/app/plugins/datasource/tempo/package.json @@ -45,7 +45,7 @@ "@testing-library/jest-dom": "6.4.1", "@testing-library/react": "14.2.1", "@testing-library/user-event": "14.5.2", - "@types/jest": "29.5.11", + "@types/jest": "29.5.12", "@types/lodash": "4.14.202", "@types/node": "20.11.13", "@types/prismjs": "1.26.3", diff --git a/yarn.lock b/yarn.lock index f8980dc2278..6a106562921 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3039,7 +3039,7 @@ __metadata: "@kusto/monaco-kusto": "npm:^7.4.0" "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/lodash": "npm:4.14.202" "@types/node": "npm:20.11.13" "@types/prismjs": "npm:1.26.3" @@ -3081,7 +3081,7 @@ __metadata: "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/debounce-promise": "npm:3.1.9" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/lodash": "npm:4.14.202" "@types/node": "npm:20.11.13" "@types/prismjs": "npm:1.26.3" @@ -3122,7 +3122,7 @@ __metadata: "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/lodash": "npm:4.14.202" "@types/prismjs": "npm:1.26.3" "@types/react": "npm:18.2.48" @@ -3163,7 +3163,7 @@ __metadata: "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" "@types/d3-random": "npm:^3.0.2" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/lodash": "npm:4.14.202" "@types/node": "npm:20.11.13" "@types/react": "npm:18.2.48" @@ -3259,7 +3259,7 @@ __metadata: "@testing-library/jest-dom": "npm:6.4.1" "@testing-library/react": "npm:14.2.1" "@testing-library/user-event": "npm:14.5.2" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/lodash": "npm:4.14.202" "@types/node": "npm:20.11.13" "@types/prismjs": "npm:1.26.3" @@ -3331,7 +3331,7 @@ __metadata: "@types/d3-interpolate": "npm:^3.0.0" "@types/dompurify": "npm:^3.0.0" "@types/history": "npm:4.7.11" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/jquery": "npm:3.5.29" "@types/lodash": "npm:4.14.202" "@types/marked": "npm:5.0.2" @@ -3703,7 +3703,7 @@ __metadata: "@testing-library/user-event": "npm:14.5.2" "@types/angular": "npm:1.8.9" "@types/history": "npm:4.7.11" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/lodash": "npm:4.14.202" "@types/react": "npm:18.2.48" "@types/react-dom": "npm:18.2.18" @@ -3867,7 +3867,7 @@ __metadata: "@types/d3": "npm:7.4.3" "@types/hoist-non-react-statics": "npm:3.3.5" "@types/is-hotkey": "npm:0.1.10" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/jquery": "npm:3.5.29" "@types/lodash": "npm:4.14.202" "@types/mock-raf": "npm:1.0.6" @@ -8874,13 +8874,13 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:*, @types/jest@npm:29.5.11, @types/jest@npm:^29.5.4": - version: 29.5.11 - resolution: "@types/jest@npm:29.5.11" +"@types/jest@npm:*, @types/jest@npm:29.5.12, @types/jest@npm:^29.5.4": + version: 29.5.12 + resolution: "@types/jest@npm:29.5.12" dependencies: expect: "npm:^29.0.0" pretty-format: "npm:^29.0.0" - checksum: 10/798f4c89407d9457bea1256de74c26f2b279f6c789c0e3311cd604cc75cdab333b9a29b00c51b0090d31abdf11cc788b4103282851a653bef6daf72edf97eef2 + checksum: 10/312e8dcf92cdd5a5847d6426f0940829bca6fe6b5a917248f3d7f7ef5d85c9ce78ef05e47d2bbabc40d41a930e0e36db2d443d2610a9e3db9062da2d5c904211 languageName: node linkType: hard @@ -17679,7 +17679,7 @@ __metadata: "@types/gtag.js": "npm:^0.0.18" "@types/history": "npm:4.7.11" "@types/hoist-non-react-statics": "npm:3.3.5" - "@types/jest": "npm:29.5.11" + "@types/jest": "npm:29.5.12" "@types/jquery": "npm:3.5.29" "@types/js-yaml": "npm:^4.0.5" "@types/jsurl": "npm:^1.2.28" From 9d17f6e6aa722ddd4b53e2cf56c421fc8ac6a313 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 1 Feb 2024 17:17:30 -0800 Subject: [PATCH 80/84] Transformers: Do not add transformation info to output frames (#81747) --- .../src/transformations/transformDataFrame.ts | 13 ------------- .../transformers/ensureColumns.test.ts | 5 ----- packages/grafana-data/src/types/data.ts | 3 --- .../extractFields/extractFields.test.ts | 3 --- .../panel/state-timeline/StateTimelinePanel.tsx | 5 +---- .../panel/state-timeline/StateTimelineTooltip2.tsx | 5 +---- .../panel/status-history/StatusHistoryPanel.tsx | 5 +---- 7 files changed, 3 insertions(+), 36 deletions(-) diff --git a/packages/grafana-data/src/transformations/transformDataFrame.ts b/packages/grafana-data/src/transformations/transformDataFrame.ts index f9baf411d6a..6717b6565a5 100644 --- a/packages/grafana-data/src/transformations/transformDataFrame.ts +++ b/packages/grafana-data/src/transformations/transformDataFrame.ts @@ -55,19 +55,6 @@ const postProcessTransform = return after; } - // Add a key to the metadata if the data changed - for (const series of after) { - if (!series.meta) { - series.meta = {}; - } - - if (!series.meta.transformations) { - series.meta.transformations = [info.id]; - } else { - series.meta.transformations = [...series.meta.transformations, info.id]; - } - } - // Add back the filtered out frames if (matcher) { // keep the frame order the same diff --git a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts index ae2b231b680..0f63477d51b 100644 --- a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts +++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts @@ -109,11 +109,6 @@ describe('ensureColumns transformer', () => { }, ], "length": 2, - "meta": { - "transformations": [ - "ensureColumns", - ], - }, } `); }); diff --git a/packages/grafana-data/src/types/data.ts b/packages/grafana-data/src/types/data.ts index 8c88d3c99eb..13a0eb789dc 100644 --- a/packages/grafana-data/src/types/data.ts +++ b/packages/grafana-data/src/types/data.ts @@ -54,9 +54,6 @@ export interface QueryResultMeta { /** Meta Notices */ notices?: QueryResultMetaNotice[]; - /** Used to track transformation ids that where part of the processing */ - transformations?: string[]; - /** Currently used to show results in Explore only in preferred visualisation option */ preferredVisualisationType?: PreferredVisualisationType; diff --git a/public/app/features/transformers/extractFields/extractFields.test.ts b/public/app/features/transformers/extractFields/extractFields.test.ts index 053a2802ba7..a6617aecea9 100644 --- a/public/app/features/transformers/extractFields/extractFields.test.ts +++ b/public/app/features/transformers/extractFields/extractFields.test.ts @@ -313,9 +313,6 @@ describe('Fields from JSON', () => { }, ], length: 2, - meta: { - transformations: ['sortBy'], - }, }; const frames = extractFieldsTransformer.transformer(extractConfig, ctx)([testDataFrame]); diff --git a/public/app/plugins/panel/state-timeline/StateTimelinePanel.tsx b/public/app/plugins/panel/state-timeline/StateTimelinePanel.tsx index 2d5020149c0..b6418ddc219 100644 --- a/public/app/plugins/panel/state-timeline/StateTimelinePanel.tsx +++ b/public/app/plugins/panel/state-timeline/StateTimelinePanel.tsx @@ -109,10 +109,7 @@ export const StateTimelinePanel = ({ * Render nothing in this case to prevent error. * See https://github.com/grafana/support-escalations/issues/932 */ - if ( - (!alignedData.meta?.transformations?.length && alignedData.fields.length - 1 !== valueFieldsCount) || - !alignedData.fields[seriesIdx] - ) { + if (alignedData.fields.length - 1 !== valueFieldsCount || !alignedData.fields[seriesIdx]) { return null; } diff --git a/public/app/plugins/panel/state-timeline/StateTimelineTooltip2.tsx b/public/app/plugins/panel/state-timeline/StateTimelineTooltip2.tsx index 187a12fb9e2..086c790e73c 100644 --- a/public/app/plugins/panel/state-timeline/StateTimelineTooltip2.tsx +++ b/public/app/plugins/panel/state-timeline/StateTimelineTooltip2.tsx @@ -68,10 +68,7 @@ export const StateTimelineTooltip2 = ({ * Render nothing in this case to prevent error. * See https://github.com/grafana/support-escalations/issues/932 */ - if ( - (!alignedData.meta?.transformations?.length && alignedData.fields.length - 1 !== valueFieldsCount) || - !alignedData.fields[seriesIdx] - ) { + if (alignedData.fields.length - 1 !== valueFieldsCount || !alignedData.fields[seriesIdx]) { return null; } diff --git a/public/app/plugins/panel/status-history/StatusHistoryPanel.tsx b/public/app/plugins/panel/status-history/StatusHistoryPanel.tsx index 95ca8ed4b3f..541e839c851 100644 --- a/public/app/plugins/panel/status-history/StatusHistoryPanel.tsx +++ b/public/app/plugins/panel/status-history/StatusHistoryPanel.tsx @@ -108,10 +108,7 @@ export const StatusHistoryPanel = ({ * Render nothing in this case to prevent error. * See https://github.com/grafana/support-escalations/issues/932 */ - if ( - (!alignedData.meta?.transformations?.length && alignedData.fields.length - 1 !== valueFieldsCount) || - !alignedData.fields[seriesIdx] - ) { + if (alignedData.fields.length - 1 !== valueFieldsCount || !alignedData.fields[seriesIdx]) { return null; } From 9c9e5e68c8bbdc30dcfe7080e5104de3a5c217b6 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 1 Feb 2024 18:14:10 -0800 Subject: [PATCH 81/84] User: Add uid colum to user table (#81615) --- packages/grafana-data/src/types/config.ts | 1 + pkg/api/dtos/models.go | 1 + pkg/api/index.go | 1 + pkg/api/user_test.go | 2 ++ .../ngalert/migration/permissions_test.go | 1 + .../migrations/accesscontrol/test/ac_test.go | 5 ++++ pkg/services/sqlstore/migrations/user_mig.go | 13 +++++++++ pkg/services/user/identity.go | 6 +++-- pkg/services/user/model.go | 7 ++++- pkg/services/user/userimpl/store.go | 5 ++++ pkg/services/user/userimpl/store_test.go | 27 +++++++++++++++++++ pkg/services/user/userimpl/user.go | 1 + public/api-enterprise-spec.json | 6 +++++ public/api-merged.json | 6 +++++ public/app/core/services/context_srv.ts | 2 ++ public/openapi3.json | 6 +++++ 16 files changed, 87 insertions(+), 3 deletions(-) diff --git a/packages/grafana-data/src/types/config.ts b/packages/grafana-data/src/types/config.ts index d6dfa26ab1f..012755695eb 100644 --- a/packages/grafana-data/src/types/config.ts +++ b/packages/grafana-data/src/types/config.ts @@ -105,6 +105,7 @@ export interface AnalyticsSettings { export interface CurrentUserDTO { isSignedIn: boolean; id: number; + uid: string; externalUserId: string; login: string; email: string; diff --git a/pkg/api/dtos/models.go b/pkg/api/dtos/models.go index c0291f65c30..8933f8b49bb 100644 --- a/pkg/api/dtos/models.go +++ b/pkg/api/dtos/models.go @@ -30,6 +30,7 @@ type LoginCommand struct { type CurrentUser struct { IsSignedIn bool `json:"isSignedIn"` Id int64 `json:"id"` + UID string `json:"uid"` Login string `json:"login"` Email string `json:"email"` Name string `json:"name"` diff --git a/pkg/api/index.go b/pkg/api/index.go index 75483655916..4c52249cbb4 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -92,6 +92,7 @@ func (hs *HTTPServer) setIndexViewData(c *contextmodel.ReqContext) (*dtos.IndexV data := dtos.IndexViewData{ User: &dtos.CurrentUser{ Id: userID, + UID: c.UserUID, // << not set yet IsSignedIn: c.IsSignedIn, Login: c.Login, Email: c.SignedInUser.GetEmail(), diff --git a/pkg/api/user_test.go b/pkg/api/user_test.go index 2b9bfa01951..964eb23da87 100644 --- a/pkg/api/user_test.go +++ b/pkg/api/user_test.go @@ -82,6 +82,7 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { } usr, err := userSvc.Create(context.Background(), &createUserCmd) require.NoError(t, err) + theUserUID := usr.UID sc.handlerFunc = hs.GetUserByID @@ -108,6 +109,7 @@ func TestUserAPIEndpoint_userLoggedIn(t *testing.T) { expected := user.UserProfileDTO{ ID: 1, + UID: theUserUID, // from original request Email: "user@test.com", Name: "user", Login: "loginuser", diff --git a/pkg/services/ngalert/migration/permissions_test.go b/pkg/services/ngalert/migration/permissions_test.go index 0b33927b60d..195b0a75200 100644 --- a/pkg/services/ngalert/migration/permissions_test.go +++ b/pkg/services/ngalert/migration/permissions_test.go @@ -643,6 +643,7 @@ func TestDashAlertPermissionMigration(t *testing.T) { for i := 1; i < 3; i++ { _, err := x.Insert(user.User{ ID: int64(i), + UID: fmt.Sprintf("u%d", i), OrgID: 1, Name: fmt.Sprintf("user%v", i), Login: fmt.Sprintf("user%v", i), diff --git a/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go b/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go index ef60e500799..c811260a588 100644 --- a/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go +++ b/pkg/services/sqlstore/migrations/accesscontrol/test/ac_test.go @@ -46,6 +46,7 @@ var ( users = []user.User{ { ID: 1, + UID: "u1", Email: "viewer1@example.org", Name: "viewer1", Login: "viewer1", @@ -55,6 +56,7 @@ var ( }, { ID: 2, + UID: "u2", Email: "viewer2@example.org", Name: "viewer2", Login: "viewer2", @@ -64,6 +66,7 @@ var ( }, { ID: 3, + UID: "u3", Email: "editor1@example.org", Name: "editor1", Login: "editor1", @@ -73,6 +76,7 @@ var ( }, { ID: 4, + UID: "u4", Email: "admin1@example.org", Name: "admin1", Login: "admin1", @@ -82,6 +86,7 @@ var ( }, { ID: 5, + UID: "u5", Email: "editor2@example.org", Name: "editor2", Login: "editor2", diff --git a/pkg/services/sqlstore/migrations/user_mig.go b/pkg/services/sqlstore/migrations/user_mig.go index 6edf5161f47..4f56666fcdb 100644 --- a/pkg/services/sqlstore/migrations/user_mig.go +++ b/pkg/services/sqlstore/migrations/user_mig.go @@ -140,6 +140,19 @@ func addUserMigrations(mg *Migrator) { SQLite(migSQLITEisServiceAccountNullable). Postgres("ALTER TABLE `user` ALTER COLUMN is_service_account DROP NOT NULL;"). Mysql("ALTER TABLE user MODIFY is_service_account BOOLEAN DEFAULT 0;")) + + mg.AddMigration("Add uid column to user", NewAddColumnMigration(userV2, &Column{ + Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: true, + })) + + mg.AddMigration("Update uid column values for users", NewRawSQLMigration(""). + SQLite("UPDATE user SET uid=printf('u%09d',id) WHERE uid IS NULL;"). + Postgres("UPDATE `user` SET uid='u' || lpad('' || id::text,9,'0') WHERE uid IS NULL;"). + Mysql("UPDATE user SET uid=concat('u',lpad(id,9,'0')) WHERE uid IS NULL;")) + + mg.AddMigration("Add unique index user_uid", NewAddIndexMigration(userV2, &Index{ + Cols: []string{"uid"}, Type: UniqueIndex, + })) } const migSQLITEisServiceAccountNullable = `ALTER TABLE user ADD COLUMN tmp_service_account BOOLEAN DEFAULT 0; diff --git a/pkg/services/user/identity.go b/pkg/services/user/identity.go index 2ab4b1f3661..119bc5e9baa 100644 --- a/pkg/services/user/identity.go +++ b/pkg/services/user/identity.go @@ -13,8 +13,9 @@ const ( ) type SignedInUser struct { - UserID int64 `xorm:"user_id"` - OrgID int64 `xorm:"org_id"` + UserID int64 `xorm:"user_id"` + UserUID string `xorm:"user_uid"` + OrgID int64 `xorm:"org_id"` OrgName string OrgRole roletype.RoleType Login string @@ -58,6 +59,7 @@ func (u *SignedInUser) NameOrFallback() string { func (u *SignedInUser) ToUserDisplayDTO() *UserDisplayDTO { return &UserDisplayDTO{ ID: u.UserID, + UID: u.UserUID, Login: u.Login, Name: u.Name, // AvatarURL: dtos.GetGravatarUrl(u.GetEmail()), diff --git a/pkg/services/user/model.go b/pkg/services/user/model.go index 81b184286ab..430f0ddccf3 100644 --- a/pkg/services/user/model.go +++ b/pkg/services/user/model.go @@ -20,7 +20,8 @@ const ( ) type User struct { - ID int64 `xorm:"pk autoincr 'id'"` + ID int64 `xorm:"pk autoincr 'id'"` + UID string `json:"uid" xorm:"uid"` Version int Email string Name string @@ -44,6 +45,7 @@ type User struct { } type CreateUserCommand struct { + UID string Email string Login string Name string @@ -115,6 +117,7 @@ type SearchUserQueryResult struct { type UserSearchHitDTO struct { ID int64 `json:"id" xorm:"id"` + UID string `json:"uid" xorm:"id"` Name string `json:"name"` Login string `json:"login"` Email string `json:"email"` @@ -133,6 +136,7 @@ type GetUserProfileQuery struct { type UserProfileDTO struct { ID int64 `json:"id"` + UID string `json:"uid"` Email string `json:"email"` Name string `json:"name"` Login string `json:"login"` @@ -215,6 +219,7 @@ type ErrCaseInsensitiveLoginConflict struct { type UserDisplayDTO struct { ID int64 `json:"id,omitempty"` + UID string `json:"uid,omitempty"` Name string `json:"name,omitempty"` Login string `json:"login,omitempty"` AvatarURL string `json:"avatarUrl"` diff --git a/pkg/services/user/userimpl/store.go b/pkg/services/user/userimpl/store.go index cedd27fe3b7..8923e446348 100644 --- a/pkg/services/user/userimpl/store.go +++ b/pkg/services/user/userimpl/store.go @@ -63,6 +63,9 @@ func (ss *sqlStore) Insert(ctx context.Context, cmd *user.User) (int64, error) { var err error err = ss.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error { sess.UseBool("is_admin") + if cmd.UID == "" { + cmd.UID = util.GenerateShortUID() + } if _, err = sess.Insert(cmd); err != nil { return err @@ -393,6 +396,7 @@ func (ss *sqlStore) GetSignedInUser(ctx context.Context, query *user.GetSignedIn var rawSQL = `SELECT u.id as user_id, + u.uid as user_uid, u.is_admin as is_grafana_admin, u.email as email, u.login as login, @@ -466,6 +470,7 @@ func (ss *sqlStore) GetProfile(ctx context.Context, query *user.GetUserProfileQu userProfile = user.UserProfileDTO{ ID: usr.ID, + UID: usr.UID, Name: usr.Name, Email: usr.Email, Login: usr.Login, diff --git a/pkg/services/user/userimpl/store_test.go b/pkg/services/user/userimpl/store_test.go index 4940bb7d15f..39d87e8d30d 100644 --- a/pkg/services/user/userimpl/store_test.go +++ b/pkg/services/user/userimpl/store_test.go @@ -104,6 +104,7 @@ func TestIntegrationUserGet(t *testing.T) { } else { require.NoError(t, err) require.NotNil(t, usr) + require.NotEmpty(t, usr.UID) } }) } @@ -150,6 +151,32 @@ func TestIntegrationUserDataAccess(t *testing.T) { require.NoError(t, err) }) + t.Run("insert user (with known UID)", func(t *testing.T) { + ctx := context.Background() + id, err := userStore.Insert(ctx, + &user.User{ + UID: "abcd", + Email: "next-test@email.com", + Name: "next-test1", + Login: "next-test1", + Created: time.Now(), + Updated: time.Now(), + }, + ) + require.NoError(t, err) + + found, err := userStore.GetByID(ctx, id) + require.NoError(t, err) + require.Equal(t, "abcd", found.UID) + + siu, err := userStore.GetSignedInUser(ctx, &user.GetSignedInUserQuery{ + UserID: id, + OrgID: found.OrgID, + }) + require.NoError(t, err) + require.Equal(t, "abcd", siu.UserUID) + }) + t.Run("get user", func(t *testing.T) { _, err := userStore.Get(context.Background(), &user.User{ diff --git a/pkg/services/user/userimpl/user.go b/pkg/services/user/userimpl/user.go index c3e64d4a8ee..eef5a1bbfb7 100644 --- a/pkg/services/user/userimpl/user.go +++ b/pkg/services/user/userimpl/user.go @@ -129,6 +129,7 @@ func (s *Service) Create(ctx context.Context, cmd *user.CreateUserCommand) (*use // create user usr := &user.User{ + UID: cmd.UID, Email: cmd.Email, Name: cmd.Name, Login: cmd.Login, diff --git a/public/api-enterprise-spec.json b/public/api-enterprise-spec.json index a7d5a6d07b1..99da38d47d5 100644 --- a/public/api-enterprise-spec.json +++ b/public/api-enterprise-spec.json @@ -7951,6 +7951,9 @@ "theme": { "type": "string" }, + "uid": { + "type": "string" + }, "updatedAt": { "type": "string", "format": "date-time" @@ -7994,6 +7997,9 @@ }, "name": { "type": "string" + }, + "uid": { + "type": "string" } } }, diff --git a/public/api-merged.json b/public/api-merged.json index 7fb4e1c5b67..5828a0d7816 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -21361,6 +21361,9 @@ "theme": { "type": "string" }, + "uid": { + "type": "string" + }, "updatedAt": { "type": "string", "format": "date-time" @@ -21404,6 +21407,9 @@ }, "name": { "type": "string" + }, + "uid": { + "type": "string" } } }, diff --git a/public/app/core/services/context_srv.ts b/public/app/core/services/context_srv.ts index 6b49da8f97e..d305d2a4fbc 100644 --- a/public/app/core/services/context_srv.ts +++ b/public/app/core/services/context_srv.ts @@ -15,6 +15,7 @@ export const AutoRefreshInterval = 'auto'; export class User implements Omit { isSignedIn: boolean; id: number; + uid: string; login: string; email: string; name: string; @@ -39,6 +40,7 @@ export class User implements Omit { constructor() { this.id = 0; + this.uid = ''; this.isGrafanaAdmin = false; this.isSignedIn = false; this.orgRole = ''; diff --git a/public/openapi3.json b/public/openapi3.json index fd6a44c65d6..5b63c944acf 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -11843,6 +11843,9 @@ "theme": { "type": "string" }, + "uid": { + "type": "string" + }, "updatedAt": { "format": "date-time", "type": "string" @@ -11886,6 +11889,9 @@ }, "name": { "type": "string" + }, + "uid": { + "type": "string" } }, "type": "object" From 7464ea43462af128d8f7965afcb837932e0632f8 Mon Sep 17 00:00:00 2001 From: Michael Mandrus <41969079+mmandrus@users.noreply.github.com> Date: Thu, 1 Feb 2024 23:52:02 -0500 Subject: [PATCH 82/84] Feature Toggles: Switch feature toggle admin page over to k8s API (#80854) * add handling for legacy and k8s apis to frontend * use backend srv directly not redux * add unit test to make sure the correct apis are being called * require api server flag * fix feature toggle name * ensure both pages work correctly * make consistent with legacy api * implement webhook update * fix unit test * remove old apis and update --------- Co-authored-by: Ryan McKinley --- pkg/api/api.go | 8 - pkg/api/featuremgmt.go | 132 ----- pkg/api/featuremgmt_test.go | 491 ------------------ pkg/api/http_server.go | 4 +- pkg/apis/featuretoggle/v0alpha1/types.go | 9 +- .../v0alpha1/zz_generated.openapi.go | 14 +- pkg/cmd/grafana/apiserver/server.go | 9 +- pkg/registry/apis/featuretoggle/current.go | 142 ++++- pkg/registry/apis/featuretoggle/register.go | 18 +- pkg/services/featuremgmt/manager.go | 12 +- pkg/services/featuremgmt/models.go | 17 +- public/app/core/reducers/root.ts | 2 - .../admin/AdminFeatureTogglesAPI.test.ts | 102 ++++ .../features/admin/AdminFeatureTogglesAPI.ts | 112 ++-- .../admin/AdminFeatureTogglesPage.tsx | 31 +- .../admin/AdminFeatureTogglesTable.tsx | 16 +- public/app/store/configureStore.ts | 4 +- 17 files changed, 348 insertions(+), 775 deletions(-) delete mode 100644 pkg/api/featuremgmt.go delete mode 100644 pkg/api/featuremgmt_test.go create mode 100644 public/app/features/admin/AdminFeatureTogglesAPI.test.ts diff --git a/pkg/api/api.go b/pkg/api/api.go index 528db6e2cab..e8886baf0bc 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -418,14 +418,6 @@ func (hs *HTTPServer) registerRoutes() { pluginRoute.Get("/:pluginId/metrics", reqOrgAdmin, routing.Wrap(hs.CollectPluginMetrics)) }) - if hs.Features.IsEnabledGlobally(featuremgmt.FlagFeatureToggleAdminPage) { - apiRoute.Group("/featuremgmt", func(featuremgmtRoute routing.RouteRegister) { - featuremgmtRoute.Get("/state", authorize(ac.EvalPermission(ac.ActionFeatureManagementRead)), hs.GetFeatureMgmtState) - featuremgmtRoute.Get("/", authorize(ac.EvalPermission(ac.ActionFeatureManagementRead)), hs.GetFeatureToggles) - featuremgmtRoute.Post("/", authorize(ac.EvalPermission(ac.ActionFeatureManagementWrite)), hs.UpdateFeatureToggle) - }) - } - apiRoute.Get("/frontend/settings/", hs.GetFrontendSettings) apiRoute.Get("/frontend/assets", hs.GetFrontendAssets) diff --git a/pkg/api/featuremgmt.go b/pkg/api/featuremgmt.go deleted file mode 100644 index 81d60909b38..00000000000 --- a/pkg/api/featuremgmt.go +++ /dev/null @@ -1,132 +0,0 @@ -package api - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "sort" - "strconv" - - "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/infra/log" - contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/web" -) - -func (hs *HTTPServer) GetFeatureToggles(ctx *contextmodel.ReqContext) response.Response { - // object being returned - dtos := make([]featuremgmt.FeatureToggleDTO, 0) - - // loop through features an add features that should be visible to dtos - for _, ft := range hs.featureManager.GetFlags() { - flag := ft.Name - if hs.featureManager.IsHiddenFromAdminPage(flag, false) { - continue - } - dto := featuremgmt.FeatureToggleDTO{ - Name: flag, - Description: ft.Description, - Enabled: hs.featureManager.IsEnabled(ctx.Req.Context(), flag), - ReadOnly: !hs.featureManager.IsEditableFromAdminPage(flag), - } - - dtos = append(dtos, dto) - sort.Slice(dtos, func(i, j int) bool { - return dtos[i].Name < dtos[j].Name - }) - } - - return response.JSON(http.StatusOK, dtos) -} - -func (hs *HTTPServer) UpdateFeatureToggle(ctx *contextmodel.ReqContext) response.Response { - featureMgmtCfg := hs.featureManager.Settings - if !featureMgmtCfg.AllowEditing { - return response.Error(http.StatusForbidden, "feature toggles are read-only", fmt.Errorf("feature toggles are configured to be read-only")) - } - - if featureMgmtCfg.UpdateWebhook == "" { - return response.Error(http.StatusInternalServerError, "feature toggles service is misconfigured", fmt.Errorf("[feature_management]update_webhook is not set")) - } - - cmd := featuremgmt.UpdateFeatureTogglesCommand{} - if err := web.Bind(ctx.Req, &cmd); err != nil { - return response.Error(http.StatusBadRequest, "bad request data", err) - } - - payload := UpdatePayload{ - FeatureToggles: make(map[string]string, len(cmd.FeatureToggles)), - User: ctx.SignedInUser.Email, - } - - for _, t := range cmd.FeatureToggles { - // make sure flag exists, and only continue if flag is writeable - if hs.featureManager.IsEditableFromAdminPage(t.Name) { - hs.log.Info("UpdateFeatureToggle: updating toggle", "toggle_name", t.Name, "enabled", t.Enabled, "username", ctx.SignedInUser.Login) - payload.FeatureToggles[t.Name] = strconv.FormatBool(t.Enabled) - } else { - hs.log.Warn("UpdateFeatureToggle: invalid toggle passed in", "toggle_name", t.Name) - return response.Error(http.StatusBadRequest, "invalid toggle passed in", fmt.Errorf("invalid toggle passed in: %s", t.Name)) - } - } - - err := sendWebhookUpdate(featureMgmtCfg, payload, hs.log) - if err != nil { - hs.log.Error("UpdateFeatureToggle: Failed to perform webhook request", "error", err) - return response.Respond(http.StatusBadRequest, "Failed to perform webhook request") - } - - hs.featureManager.SetRestartRequired() - - return response.Respond(http.StatusOK, "feature toggles updated successfully") -} - -func (hs *HTTPServer) GetFeatureMgmtState(ctx *contextmodel.ReqContext) response.Response { - fmState := hs.featureManager.GetState() - return response.Respond(http.StatusOK, fmState) -} - -type UpdatePayload struct { - FeatureToggles map[string]string `json:"feature_toggles"` - User string `json:"user"` -} - -func sendWebhookUpdate(cfg setting.FeatureMgmtSettings, payload UpdatePayload, logger log.Logger) error { - data, err := json.Marshal(payload) - if err != nil { - return err - } - - req, err := http.NewRequest(http.MethodPost, cfg.UpdateWebhook, bytes.NewBuffer(data)) - if err != nil { - return err - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+cfg.UpdateWebhookToken) - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return err - } - defer func() { - if err := resp.Body.Close(); err != nil { - logger.Warn("Failed to close response body", "err", err) - } - }() - - if resp.StatusCode >= http.StatusBadRequest { - if body, err := io.ReadAll(resp.Body); err != nil { - return fmt.Errorf("SendWebhookUpdate failed with status=%d, error: %s", resp.StatusCode, string(body)) - } else { - return fmt.Errorf("SendWebhookUpdate failed with status=%d, error: %w", resp.StatusCode, err) - } - } - - return nil -} diff --git a/pkg/api/featuremgmt_test.go b/pkg/api/featuremgmt_test.go deleted file mode 100644 index bea1b4c089c..00000000000 --- a/pkg/api/featuremgmt_test.go +++ /dev/null @@ -1,491 +0,0 @@ -package api - -import ( - "bytes" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/org/orgtest" - "github.com/grafana/grafana/pkg/services/user" - "github.com/grafana/grafana/pkg/services/user/usertest" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/web/webtest" -) - -func TestGetFeatureToggles(t *testing.T) { - readPermissions := []accesscontrol.Permission{{Action: accesscontrol.ActionFeatureManagementRead}} - - t.Run("should not be able to get feature toggles without permissions", func(t *testing.T) { - result := runGetScenario(t, []*featuremgmt.FeatureFlag{}, []string{}, setting.FeatureMgmtSettings{}, []accesscontrol.Permission{}, http.StatusForbidden) - assert.Len(t, result, 0) - }) - - t.Run("should be able to get feature toggles with correct permissions", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: "toggle1", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, - } - disabled := []string{"toggle2"} - - result := runGetScenario(t, features, disabled, setting.FeatureMgmtSettings{}, readPermissions, http.StatusOK) - assert.Len(t, result, 2) - t1, _ := findResult(t, result, "toggle1") - assert.True(t, t1.Enabled) - t2, _ := findResult(t, result, "toggle2") - assert.False(t, t2.Enabled) - }) - - t.Run("toggles hidden by config are not present in the response", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: "toggle1", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, - } - settings := setting.FeatureMgmtSettings{ - HiddenToggles: map[string]struct{}{"toggle1": {}}, - } - - result := runGetScenario(t, features, []string{}, settings, readPermissions, http.StatusOK) - assert.Len(t, result, 1) - assert.Equal(t, "toggle2", result[0].Name) - }) - - t.Run("toggles that are read-only by config have the readOnly field set", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: "toggle1", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, - } - disabled := []string{"toggle2"} - settings := setting.FeatureMgmtSettings{ - HiddenToggles: map[string]struct{}{"toggle1": {}}, - ReadOnlyToggles: map[string]struct{}{"toggle2": {}}, - AllowEditing: true, - UpdateWebhook: "bogus", - } - - result := runGetScenario(t, features, disabled, settings, readPermissions, http.StatusOK) - assert.Len(t, result, 1) - assert.Equal(t, "toggle2", result[0].Name) - assert.True(t, result[0].ReadOnly) - }) - - t.Run("feature toggle defailts", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: "toggle1", - Stage: featuremgmt.FeatureStageUnknown, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStageExperimental, - }, { - Name: "toggle3", - Stage: featuremgmt.FeatureStagePrivatePreview, - }, { - Name: "toggle4", - Stage: featuremgmt.FeatureStagePublicPreview, - AllowSelfServe: true, - }, { - Name: "toggle5", - Stage: featuremgmt.FeatureStageGeneralAvailability, - AllowSelfServe: true, - }, { - Name: "toggle6", - Stage: featuremgmt.FeatureStageDeprecated, - AllowSelfServe: true, - }, { - Name: "toggle7", - Stage: featuremgmt.FeatureStageGeneralAvailability, - AllowSelfServe: false, - }, - } - - t.Run("unknown, experimental, and private preview toggles are hidden by default", func(t *testing.T) { - result := runGetScenario(t, features, []string{}, setting.FeatureMgmtSettings{}, readPermissions, http.StatusOK) - assert.Len(t, result, 4) - - _, ok := findResult(t, result, "toggle1") - assert.False(t, ok) - _, ok = findResult(t, result, "toggle2") - assert.False(t, ok) - _, ok = findResult(t, result, "toggle3") - assert.False(t, ok) - }) - - t.Run("only public preview and GA with AllowSelfServe are writeable", func(t *testing.T) { - settings := setting.FeatureMgmtSettings{ - AllowEditing: true, - UpdateWebhook: "bogus", - } - result := runGetScenario(t, features, []string{}, settings, readPermissions, http.StatusOK) - assert.Len(t, result, 4) - - t4, ok := findResult(t, result, "toggle4") - assert.True(t, ok) - assert.True(t, t4.ReadOnly) - t5, ok := findResult(t, result, "toggle5") - assert.True(t, ok) - assert.False(t, t5.ReadOnly) - t6, ok := findResult(t, result, "toggle6") - assert.True(t, ok) - assert.False(t, t6.ReadOnly) - }) - - t.Run("all toggles are read-only when server is misconfigured", func(t *testing.T) { - settings := setting.FeatureMgmtSettings{ - AllowEditing: false, - UpdateWebhook: "", - } - result := runGetScenario(t, features, []string{}, settings, readPermissions, http.StatusOK) - assert.Len(t, result, 4) - - t4, ok := findResult(t, result, "toggle4") - assert.True(t, ok) - assert.True(t, t4.ReadOnly) - t5, ok := findResult(t, result, "toggle5") - assert.True(t, ok) - assert.True(t, t5.ReadOnly) - t6, ok := findResult(t, result, "toggle6") - assert.True(t, ok) - assert.True(t, t6.ReadOnly) - }) - }) -} - -func TestSetFeatureToggles(t *testing.T) { - writePermissions := []accesscontrol.Permission{{Action: accesscontrol.ActionFeatureManagementWrite}} - - t.Run("fails without adequate permissions", func(t *testing.T) { - res := runSetScenario(t, nil, []string{}, nil, setting.FeatureMgmtSettings{}, []accesscontrol.Permission{}, http.StatusForbidden) - defer func() { require.NoError(t, res.Body.Close()) }() - }) - - t.Run("fails when toggle editing is not enabled", func(t *testing.T) { - res := runSetScenario(t, nil, []string{}, nil, setting.FeatureMgmtSettings{}, writePermissions, http.StatusForbidden) - defer func() { require.NoError(t, res.Body.Close()) }() - p := readBody(t, res.Body) - assert.Equal(t, "feature toggles are read-only", p["message"]) - }) - - t.Run("fails when update toggle url is not set", func(t *testing.T) { - s := setting.FeatureMgmtSettings{ - AllowEditing: true, - } - res := runSetScenario(t, nil, []string{}, nil, s, writePermissions, http.StatusInternalServerError) - defer func() { require.NoError(t, res.Body.Close()) }() - p := readBody(t, res.Body) - assert.Equal(t, "feature toggles service is misconfigured", p["message"]) - }) - - t.Run("fails with non-existent toggle", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: "toggle1", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, - } - disabled := []string{"toggle2"} - updates := []featuremgmt.FeatureToggleDTO{ - { - Name: "toggle3", - Enabled: true, - }, - } - - s := setting.FeatureMgmtSettings{ - AllowEditing: true, - UpdateWebhook: "random", - } - res := runSetScenario(t, features, disabled, updates, s, writePermissions, http.StatusBadRequest) - defer func() { require.NoError(t, res.Body.Close()) }() - p := readBody(t, res.Body) - assert.Equal(t, "invalid toggle passed in", p["message"]) - }) - - t.Run("fails with read-only toggles", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: featuremgmt.FlagFeatureToggleAdminPage, - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStagePublicPreview, - }, { - Name: "toggle3", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, - } - disabled := []string{"toggle2", "toggle3"} - - s := setting.FeatureMgmtSettings{ - AllowEditing: true, - UpdateWebhook: "random", - ReadOnlyToggles: map[string]struct{}{ - "toggle3": {}, - }, - } - - t.Run("because it is the feature toggle admin page toggle", func(t *testing.T) { - updates := []featuremgmt.FeatureToggleDTO{ - { - Name: featuremgmt.FlagFeatureToggleAdminPage, - Enabled: true, - }, - } - res := runSetScenario(t, features, disabled, updates, s, writePermissions, http.StatusBadRequest) - defer func() { require.NoError(t, res.Body.Close()) }() - p := readBody(t, res.Body) - assert.Equal(t, "invalid toggle passed in", p["message"]) - }) - - t.Run("because it is not GA or Deprecated", func(t *testing.T) { - updates := []featuremgmt.FeatureToggleDTO{ - { - Name: "toggle2", - Enabled: true, - }, - } - res := runSetScenario(t, features, disabled, updates, s, writePermissions, http.StatusBadRequest) - defer func() { require.NoError(t, res.Body.Close()) }() - p := readBody(t, res.Body) - assert.Equal(t, "invalid toggle passed in", p["message"]) - }) - - t.Run("because it is configured to be read-only", func(t *testing.T) { - updates := []featuremgmt.FeatureToggleDTO{ - { - Name: "toggle3", - Enabled: true, - }, - } - res := runSetScenario(t, features, disabled, updates, s, writePermissions, http.StatusBadRequest) - defer func() { require.NoError(t, res.Body.Close()) }() - p := readBody(t, res.Body) - assert.Equal(t, "invalid toggle passed in", p["message"]) - }) - }) - - t.Run("when all conditions met", func(t *testing.T) { - features := []*featuremgmt.FeatureFlag{ - { - Name: featuremgmt.FlagFeatureToggleAdminPage, - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle2", - Stage: featuremgmt.FeatureStagePublicPreview, - }, { - Name: "toggle3", - Stage: featuremgmt.FeatureStageGeneralAvailability, - }, { - Name: "toggle4", - Stage: featuremgmt.FeatureStageGeneralAvailability, - AllowSelfServe: true, - }, { - Name: "toggle5", - Stage: featuremgmt.FeatureStageDeprecated, - AllowSelfServe: true, - }, - } - disabled := []string{"toggle2", "toggle3", "toggle4", "toggle5"} - - s := setting.FeatureMgmtSettings{ - AllowEditing: true, - UpdateWebhook: "random", - UpdateWebhookToken: "token", - ReadOnlyToggles: map[string]struct{}{ - "toggle3": {}, - }, - } - - updates := []featuremgmt.FeatureToggleDTO{ - { - Name: "toggle4", - Enabled: true, - }, { - Name: "toggle5", - Enabled: false, - }, - } - t.Run("fail when webhook request is not successful", func(t *testing.T) { - webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - })) - defer webhookServer.Close() - s.UpdateWebhook = webhookServer.URL - res := runSetScenario(t, features, disabled, updates, s, writePermissions, http.StatusBadRequest) - defer func() { require.NoError(t, res.Body.Close()) }() - assert.Equal(t, http.StatusBadRequest, res.StatusCode) - }) - - t.Run("succeed when webhook request is successul", func(t *testing.T) { - webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "Bearer "+s.UpdateWebhookToken, r.Header.Get("Authorization")) - - var req UpdatePayload - require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) - - assert.Equal(t, "true", req.FeatureToggles["toggle4"]) - assert.Equal(t, "false", req.FeatureToggles["toggle5"]) - w.WriteHeader(http.StatusOK) - })) - defer webhookServer.Close() - s.UpdateWebhook = webhookServer.URL - res := runSetScenario(t, features, disabled, updates, s, writePermissions, http.StatusOK) - defer func() { require.NoError(t, res.Body.Close()) }() - assert.Equal(t, http.StatusOK, res.StatusCode) - }) - }) -} - -func findResult(t *testing.T, result []featuremgmt.FeatureToggleDTO, name string) (featuremgmt.FeatureToggleDTO, bool) { - t.Helper() - - for _, t := range result { - if t.Name == name { - return t, true - } - } - return featuremgmt.FeatureToggleDTO{}, false -} - -func readBody(t *testing.T, rc io.ReadCloser) map[string]any { - t.Helper() - - b, err := io.ReadAll(rc) - require.NoError(t, err) - payload := map[string]any{} - require.NoError(t, json.Unmarshal(b, &payload)) - return payload -} - -func runGetScenario( - t *testing.T, - features []*featuremgmt.FeatureFlag, - disabled []string, // the flags that are disabled - settings setting.FeatureMgmtSettings, - permissions []accesscontrol.Permission, - expectedCode int, -) []featuremgmt.FeatureToggleDTO { - // Set up server and send request - cfg := setting.NewCfg() - - fm := featuremgmt.WithFeatureManager(settings, append([]*featuremgmt.FeatureFlag{{ - Name: featuremgmt.FlagFeatureToggleAdminPage, - Stage: featuremgmt.FeatureStageGeneralAvailability, - }}, features...), disabled...) - - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = cfg - hs.Features = fm - hs.featureManager = fm - hs.orgService = orgtest.NewOrgServiceFake() - hs.userService = &usertest.FakeUserService{ - ExpectedUser: &user.User{ID: 1}, - } - hs.log = log.New("test") - }) - req := webtest.RequestWithSignedInUser(server.NewGetRequest("/api/featuremgmt"), userWithPermissions(1, permissions)) - res, err := server.SendJSON(req) - defer func() { require.NoError(t, res.Body.Close()) }() - - // Do some general checks for every request - require.NoError(t, err) - require.Equal(t, expectedCode, res.StatusCode) - if res.StatusCode >= 400 { - return nil - } - - var result []featuremgmt.FeatureToggleDTO - err = json.NewDecoder(res.Body).Decode(&result) - require.NoError(t, err) - - for i := 0; i < len(result); { - ft := result[i] - // Always make sure admin page toggle is read-only, then remove it to make assertions easier - if ft.Name == featuremgmt.FlagFeatureToggleAdminPage { - assert.True(t, ft.ReadOnly) - result = append(result[:i], result[i+1:]...) - continue - } - - // Make sure toggles explicitly marked "hidden" by config are hidden - if _, ok := cfg.FeatureManagement.HiddenToggles[ft.Name]; ok { - t.Fail() - } - - // Make sure toggles explicitly marked "read only" by config are read only - if _, ok := cfg.FeatureManagement.ReadOnlyToggles[ft.Name]; ok { - assert.True(t, ft.ReadOnly) - } - i++ - } - - return result -} - -func runSetScenario( - t *testing.T, - serverFeatures []*featuremgmt.FeatureFlag, - disabled []string, // the flags that are disabled - updateFeatures []featuremgmt.FeatureToggleDTO, - settings setting.FeatureMgmtSettings, - permissions []accesscontrol.Permission, - expectedCode int, -) *http.Response { - // Set up server and send request - cfg := setting.NewCfg() - features := featuremgmt.WithFeatureManager(settings, append([]*featuremgmt.FeatureFlag{{ - Name: featuremgmt.FlagFeatureToggleAdminPage, - Stage: featuremgmt.FeatureStageGeneralAvailability, - }}, serverFeatures...), disabled...) - - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = cfg - hs.Features = features - hs.featureManager = features - hs.orgService = orgtest.NewOrgServiceFake() - hs.userService = &usertest.FakeUserService{ - ExpectedUser: &user.User{ID: 1}, - } - hs.log = log.New("test") - }) - - cmd := featuremgmt.UpdateFeatureTogglesCommand{ - FeatureToggles: updateFeatures, - } - b, err := json.Marshal(cmd) - require.NoError(t, err) - req := webtest.RequestWithSignedInUser(server.NewPostRequest("/api/featuremgmt", bytes.NewReader(b)), userWithPermissions(1, permissions)) - res, err := server.SendJSON(req) - - require.NoError(t, err) - require.NotNil(t, res) - require.Equal(t, expectedCode, res.StatusCode) - - return res -} diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index c9a4536f2a1..c7cce87d03f 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -127,7 +127,6 @@ type HTTPServer struct { RenderService rendering.Service Cfg *setting.Cfg Features featuremgmt.FeatureToggles - featureManager *featuremgmt.FeatureManager SettingsProvider setting.Provider HooksService *hooks.HooksService navTreeService navtree.Service @@ -235,7 +234,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi correlationsService correlations.Service, remoteCache *remotecache.RemoteCache, provisioningService provisioning.ProvisioningService, accessControl accesscontrol.AccessControl, dataSourceProxy *datasourceproxy.DataSourceProxyService, searchService *search.SearchService, live *live.GrafanaLive, livePushGateway *pushhttp.Gateway, plugCtxProvider *plugincontext.Provider, - contextHandler *contexthandler.ContextHandler, loggerMiddleware loggermw.Logger, features *featuremgmt.FeatureManager, + contextHandler *contexthandler.ContextHandler, loggerMiddleware loggermw.Logger, features featuremgmt.FeatureToggles, alertNG *ngalert.AlertNG, libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service, quotaService quota.Service, socialService social.Service, tracer tracing.Tracer, encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService, @@ -292,7 +291,6 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi QueryHistoryService: queryHistoryService, CorrelationsService: correlationsService, Features: features, // a read only view of the managers state - featureManager: features, StorageService: storageService, RemoteCacheService: remoteCache, ProvisioningService: provisioningService, diff --git a/pkg/apis/featuretoggle/v0alpha1/types.go b/pkg/apis/featuretoggle/v0alpha1/types.go index 316145befd7..4fa25c5135d 100644 --- a/pkg/apis/featuretoggle/v0alpha1/types.go +++ b/pkg/apis/featuretoggle/v0alpha1/types.go @@ -78,8 +78,11 @@ type FeatureTogglesList struct { type ResolvedToggleState struct { metav1.TypeMeta `json:",inline"` - // Can any flag be updated - Writeable bool `json:"writeable,omitempty"` + // The user is allowed to edit feature toggles on this system + AllowEditing bool `json:"allowEditing,omitempty"` + + // The system has changes that require still require a restart + RestartRequired bool `json:"restartRequired,omitempty"` // The currently enabled flags Enabled map[string]bool `json:"enabled,omitempty"` @@ -99,7 +102,7 @@ type ToggleStatus struct { Enabled bool `json:"enabled"` // Can this flag be updated - Writeable bool `json:"writeable,omitempty"` + Writeable bool `json:"writeable"` // Where was the value configured // eg: startup | tenant|org | user | browser diff --git a/pkg/apis/featuretoggle/v0alpha1/zz_generated.openapi.go b/pkg/apis/featuretoggle/v0alpha1/zz_generated.openapi.go index 3e9c9364b32..94ddee37f44 100644 --- a/pkg/apis/featuretoggle/v0alpha1/zz_generated.openapi.go +++ b/pkg/apis/featuretoggle/v0alpha1/zz_generated.openapi.go @@ -318,9 +318,16 @@ func schema_pkg_apis_featuretoggle_v0alpha1_ResolvedToggleState(ref common.Refer Format: "", }, }, - "writeable": { + "allowEditing": { SchemaProps: spec.SchemaProps{ - Description: "Can any flag be updated", + Description: "The user is allowed to edit feature toggles on this system", + Type: []string{"boolean"}, + Format: "", + }, + }, + "restartRequired": { + SchemaProps: spec.SchemaProps{ + Description: "The system has changes that require still require a restart", Type: []string{"boolean"}, Format: "", }, @@ -395,6 +402,7 @@ func schema_pkg_apis_featuretoggle_v0alpha1_ToggleStatus(ref common.ReferenceCal "writeable": { SchemaProps: spec.SchemaProps{ Description: "Can this flag be updated", + Default: false, Type: []string{"boolean"}, Format: "", }, @@ -413,7 +421,7 @@ func schema_pkg_apis_featuretoggle_v0alpha1_ToggleStatus(ref common.ReferenceCal }, }, }, - Required: []string{"name", "enabled"}, + Required: []string{"name", "enabled", "writeable"}, }, }, Dependencies: []string{ diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index a8f5fffb3e7..cd36da5abda 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/query" "github.com/grafana/grafana/pkg/registry/apis/query/runner" "github.com/grafana/grafana/pkg/server" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/apiserver/utils" @@ -61,8 +62,12 @@ func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error { runner.NewDummyRegistry(), )) case "featuretoggle.grafana.app": - features := featuremgmt.WithFeatureManager(setting.FeatureMgmtSettings{}, nil) // none... for now - o.builders = append(o.builders, featuretoggle.NewFeatureFlagAPIBuilder(features)) + o.builders = append(o.builders, + featuretoggle.NewFeatureFlagAPIBuilder( + featuremgmt.WithFeatureManager(setting.FeatureMgmtSettings{}, nil), // none... for now + &actest.FakeAccessControl{ExpectedEvaluate: false}, + ), + ) case "testdata.datasource.grafana.app": ds, err := server.InitializeDataSourceAPIServer(g) if err != nil { diff --git a/pkg/registry/apis/featuretoggle/current.go b/pkg/registry/apis/featuretoggle/current.go index 6813be4d0de..7bc5661c75b 100644 --- a/pkg/registry/apis/featuretoggle/current.go +++ b/pkg/registry/apis/featuretoggle/current.go @@ -1,25 +1,37 @@ package featuretoggle import ( + "bytes" "context" "encoding/json" + "fmt" + "io" "net/http" + "strconv" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/infra/appcontext" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util/errutil" + "github.com/grafana/grafana/pkg/util/errutil/errhttp" "github.com/grafana/grafana/pkg/web" ) -func getResolvedToggleState(ctx context.Context, features *featuremgmt.FeatureManager) v0alpha1.ResolvedToggleState { +func (b *FeatureFlagAPIBuilder) getResolvedToggleState(ctx context.Context) v0alpha1.ResolvedToggleState { state := v0alpha1.ResolvedToggleState{ TypeMeta: v1.TypeMeta{ APIVersion: v0alpha1.APIVERSION, Kind: "ResolvedToggleState", }, - Enabled: features.GetEnabled(ctx), + Enabled: b.features.GetEnabled(ctx), + RestartRequired: b.features.IsRestartRequired(), } // Reference to the object that defined the values @@ -28,11 +40,11 @@ func getResolvedToggleState(ctx context.Context, features *featuremgmt.FeatureMa Name: "startup", } - startup := features.GetStartupFlags() - warnings := features.GetWarning() - for _, f := range features.GetFlags() { + startup := b.features.GetStartupFlags() + warnings := b.features.GetWarning() + for _, f := range b.features.GetFlags() { name := f.Name - if features.IsHiddenFromAdminPage(name, true) { + if b.features.IsHiddenFromAdminPage(name, false) { continue } @@ -40,7 +52,7 @@ func getResolvedToggleState(ctx context.Context, features *featuremgmt.FeatureMa Name: name, Description: f.Description, // simplify the UI changes Enabled: state.Enabled[name], - Writeable: features.IsEditableFromAdminPage(name), + Writeable: b.features.IsEditableFromAdminPage(name), Source: startupRef, Warning: warnings[name], } @@ -51,55 +63,87 @@ func getResolvedToggleState(ctx context.Context, features *featuremgmt.FeatureMa if toggle.Enabled || toggle.Writeable || toggle.Warning != "" || inStartup { state.Toggles = append(state.Toggles, toggle) } + + if toggle.Writeable { + state.AllowEditing = true + } + } + + // Make sure the user can actually write values + if state.AllowEditing { + state.AllowEditing = b.features.IsFeatureEditingAllowed() && b.userCanWrite(ctx, nil) } return state } +func (b *FeatureFlagAPIBuilder) userCanWrite(ctx context.Context, u *user.SignedInUser) bool { + if u == nil { + u, _ = appcontext.User(ctx) + if u == nil { + return false + } + } + ok, err := b.accessControl.Evaluate(ctx, u, ac.EvalPermission(ac.ActionFeatureManagementWrite)) + return ok && err == nil +} + func (b *FeatureFlagAPIBuilder) handleCurrentStatus(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPatch { b.handlePatchCurrent(w, r) return } - state := getResolvedToggleState(r.Context(), b.features) - - err := json.NewEncoder(w).Encode(state) - if err != nil { - w.WriteHeader(500) - } + state := b.getResolvedToggleState(r.Context()) + _ = json.NewEncoder(w).Encode(state) } // NOTE: authz is already handled by the authorizer func (b *FeatureFlagAPIBuilder) handlePatchCurrent(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() if !b.features.IsFeatureEditingAllowed() { - _, _ = w.Write([]byte("Feature editing is disabled")) + errhttp.Write(ctx, fmt.Errorf("feature editing is not enabled"), w) return } - ctx := r.Context() - request := v0alpha1.ResolvedToggleState{} - err := web.Bind(r, &request) + user, err := appcontext.User(ctx) if err != nil { - _, _ = w.Write([]byte("ERROR!!! " + err.Error())) + errhttp.Write(ctx, err, w) + return + } + + if !b.userCanWrite(ctx, user) { + err = errutil.Unauthorized("featuretoggle.canNotWrite", + errutil.WithPublicMessage("missing write permission")) + errhttp.Write(ctx, err, w) + return + } + + request := v0alpha1.ResolvedToggleState{} + err = web.Bind(r, &request) + if err != nil { + errhttp.Write(ctx, err, w) return } if len(request.Toggles) > 0 { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte("can only patch the enabled")) + err = errutil.BadRequest("featuretoggle.badRequest", + errutil.WithPublicMessage("can only path the enabled section")) + errhttp.Write(ctx, err, w) return } - changes := map[string]bool{} + changes := map[string]string{} // TODO would be nice to have this be a bool on the HG side for k, v := range request.Enabled { current := b.features.IsEnabled(ctx, k) if current != v { if !b.features.IsEditableFromAdminPage(k) { + err = errutil.BadRequest("featuretoggle.badRequest", + errutil.WithPublicMessage("can not edit toggle: "+k)) + errhttp.Write(ctx, err, w) w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte("can not edit toggle: " + k)) return } - changes[k] = v + changes[k] = strconv.FormatBool(v) } } @@ -108,5 +152,55 @@ func (b *FeatureFlagAPIBuilder) handlePatchCurrent(w http.ResponseWriter, r *htt return } - _, _ = w.Write([]byte("TODO... actually UPDATE/call webhook: ")) + payload := featuremgmt.FeatureToggleWebhookPayload{ + FeatureToggles: changes, + User: user.Email, + } + + err = sendWebhookUpdate(b.features.Settings, payload) + if err != nil { + errhttp.Write(ctx, err, w) + return + } + + b.features.SetRestartRequired() + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("feature toggles updated successfully")) +} + +func sendWebhookUpdate(cfg setting.FeatureMgmtSettings, payload featuremgmt.FeatureToggleWebhookPayload) error { + data, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, cfg.UpdateWebhook, bytes.NewBuffer(data)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+cfg.UpdateWebhookToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logger.Warn("Failed to close response body", "err", err) + } + }() + + if resp.StatusCode >= http.StatusBadRequest { + if body, err := io.ReadAll(resp.Body); err != nil { + return fmt.Errorf("SendWebhookUpdate failed with status=%d, error: %s", resp.StatusCode, string(body)) + } else { + return fmt.Errorf("SendWebhookUpdate failed with status=%d, error: %w", resp.StatusCode, err) + } + } + + return nil } diff --git a/pkg/registry/apis/featuretoggle/register.go b/pkg/registry/apis/featuretoggle/register.go index 080bb42f20e..f5ae2466c42 100644 --- a/pkg/registry/apis/featuretoggle/register.go +++ b/pkg/registry/apis/featuretoggle/register.go @@ -14,6 +14,7 @@ import ( "k8s.io/kube-openapi/pkg/validation/spec" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -24,20 +25,19 @@ var gv = v0alpha1.SchemeGroupVersion // This is used just so wire has something unique to return type FeatureFlagAPIBuilder struct { - features *featuremgmt.FeatureManager + features *featuremgmt.FeatureManager + accessControl accesscontrol.AccessControl } -func NewFeatureFlagAPIBuilder(features *featuremgmt.FeatureManager) *FeatureFlagAPIBuilder { - return &FeatureFlagAPIBuilder{features} +func NewFeatureFlagAPIBuilder(features *featuremgmt.FeatureManager, accessControl accesscontrol.AccessControl) *FeatureFlagAPIBuilder { + return &FeatureFlagAPIBuilder{features, accessControl} } func RegisterAPIService(features *featuremgmt.FeatureManager, + accessControl accesscontrol.AccessControl, apiregistration builder.APIRegistrar, ) *FeatureFlagAPIBuilder { - if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { - return nil // skip registration unless opting into experimental apis - } - builder := NewFeatureFlagAPIBuilder(features) + builder := NewFeatureFlagAPIBuilder(features, accessControl) apiregistration.RegisterAPI(builder) return builder } @@ -161,7 +161,7 @@ func (b *FeatureFlagAPIBuilder) GetAPIRoutes() *builder.APIRoutes { "enable-auto-migrate": { ExampleProps: spec3.ExampleProps{ Summary: "enable auto-migrate panels", - Description: "example descr", + Description: "enable description", Value: &v0alpha1.ResolvedToggleState{ Enabled: map[string]bool{ featuremgmt.FlagAutoMigrateOldPanels: true, @@ -172,7 +172,7 @@ func (b *FeatureFlagAPIBuilder) GetAPIRoutes() *builder.APIRoutes { "disable-auto-migrate": { ExampleProps: spec3.ExampleProps{ Summary: "disable auto-migrate panels", - Description: "disable descr", + Description: "disable description", Value: &v0alpha1.ResolvedToggleState{ Enabled: map[string]bool{ featuremgmt.FlagAutoMigrateOldPanels: false, diff --git a/pkg/services/featuremgmt/manager.go b/pkg/services/featuremgmt/manager.go index 80de907ff77..74ea3350412 100644 --- a/pkg/services/featuremgmt/manager.go +++ b/pkg/services/featuremgmt/manager.go @@ -131,18 +131,16 @@ func (fm *FeatureManager) GetFlags() []FeatureFlag { return v } -func (fm *FeatureManager) GetState() *FeatureManagerState { - return &FeatureManagerState{ - RestartRequired: fm.restartRequired, - AllowEditing: fm.Settings.AllowEditing, - } -} - // isFeatureEditingAllowed checks if the backend is properly configured to allow feature toggle changes from the UI func (fm *FeatureManager) IsFeatureEditingAllowed() bool { return fm.Settings.AllowEditing && fm.Settings.UpdateWebhook != "" } +// indicate if a change has been made (not that accurate, but better than nothing) +func (fm *FeatureManager) IsRestartRequired() bool { + return fm.restartRequired +} + // Flags that can be edited func (fm *FeatureManager) IsEditableFromAdminPage(key string) bool { flag, ok := fm.flags[key] diff --git a/pkg/services/featuremgmt/models.go b/pkg/services/featuremgmt/models.go index 49d6a353b3b..b9c8b9ff7ea 100644 --- a/pkg/services/featuremgmt/models.go +++ b/pkg/services/featuremgmt/models.go @@ -134,18 +134,7 @@ type FeatureFlag struct { RequiresRestart bool `json:"requiresRestart,omitempty"` } -type UpdateFeatureTogglesCommand struct { - FeatureToggles []FeatureToggleDTO `json:"featureToggles"` -} - -type FeatureToggleDTO struct { - Name string `json:"name" binding:"Required"` - Description string `json:"description"` - Enabled bool `json:"enabled"` - ReadOnly bool `json:"readOnly,omitempty"` -} - -type FeatureManagerState struct { - RestartRequired bool `json:"restartRequired"` - AllowEditing bool `json:"allowEditing"` +type FeatureToggleWebhookPayload struct { + FeatureToggles map[string]string `json:"feature_toggles"` + User string `json:"user"` } diff --git a/public/app/core/reducers/root.ts b/public/app/core/reducers/root.ts index 9ab94dbdf07..f2c436062bd 100644 --- a/public/app/core/reducers/root.ts +++ b/public/app/core/reducers/root.ts @@ -2,7 +2,6 @@ import { ReducersMapObject } from '@reduxjs/toolkit'; import { AnyAction, combineReducers } from 'redux'; import sharedReducers from 'app/core/reducers'; -import { togglesApi } from 'app/features/admin/AdminFeatureTogglesAPI'; import ldapReducers from 'app/features/admin/state/reducers'; import alertingReducers from 'app/features/alerting/state/reducers'; import apiKeysReducers from 'app/features/api-keys/state/reducers'; @@ -56,7 +55,6 @@ const rootReducers = { [alertingApi.reducerPath]: alertingApi.reducer, [publicDashboardApi.reducerPath]: publicDashboardApi.reducer, [browseDashboardsAPI.reducerPath]: browseDashboardsAPI.reducer, - [togglesApi.reducerPath]: togglesApi.reducer, }; const addedReducers = {}; diff --git a/public/app/features/admin/AdminFeatureTogglesAPI.test.ts b/public/app/features/admin/AdminFeatureTogglesAPI.test.ts new file mode 100644 index 00000000000..ca6c25be525 --- /dev/null +++ b/public/app/features/admin/AdminFeatureTogglesAPI.test.ts @@ -0,0 +1,102 @@ +import { BackendSrvRequest, config } from '@grafana/runtime'; + +import { getTogglesAPI } from './AdminFeatureTogglesAPI'; + +// implements @grafana/runtime/BackendSrv +class MockSrv { + constructor() { + this.apiCalls = []; + } + + apiCalls: Array<{ + url: string; + method: string; + }>; + + async get( + url: string, + params?: BackendSrvRequest['params'], + requestId?: BackendSrvRequest['requestId'], + options?: Partial + ) { + this.apiCalls.push({ + url: url, + method: 'get', + }); + if (config.featureToggles.kubernetesFeatureToggles && url.indexOf('current') > -1) { + return await { toggles: [] }; + } + + return await {}; + } + + async post(url: string, data?: unknown, options?: Partial) { + this.apiCalls.push({ + url: url, + method: 'post', + }); + return await {}; + } + + async patch(url: string, data: unknown, options?: Partial) { + this.apiCalls.push({ + url: url, + method: 'patch', + }); + return await {}; + } + + // these aren't needed for this test + async put(url: string, data: unknown, options?: Partial) { + return await {}; + } + async delete(url: string, data?: unknown, options?: Partial) { + return await {}; + } +} + +const testBackendSrv = new MockSrv(); + +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getBackendSrv: () => testBackendSrv, + config: { + featureToggles: { + kubernetesFeatureToggles: false, + grafanaAPIServerWithExperimentalAPIs: false, + }, + }, +})); + +describe('AdminFeatureTogglesApi', () => { + beforeEach(() => { + jest.clearAllMocks(); + testBackendSrv.apiCalls.length = 0; + }); + + const originalToggles = { ...config.featureToggles }; + + afterAll(() => { + config.featureToggles = originalToggles; + }); + + it('uses the k8s api when the k8s toggles are on', async () => { + config.featureToggles.kubernetesFeatureToggles = true; + config.featureToggles.grafanaAPIServerWithExperimentalAPIs = true; + + const togglesApi = getTogglesAPI(); + await togglesApi.getFeatureToggles(); + await togglesApi.updateFeatureToggles([]); + const expected = [ + { + method: 'get', + url: '/apis/featuretoggle.grafana.app/v0alpha1/current', + }, + { + method: 'patch', + url: '/apis/featuretoggle.grafana.app/v0alpha1/current', + }, + ]; + expect(testBackendSrv.apiCalls).toEqual(expect.arrayContaining(expected)); + }); +}); diff --git a/public/app/features/admin/AdminFeatureTogglesAPI.ts b/public/app/features/admin/AdminFeatureTogglesAPI.ts index 889f6f0759e..62f0c8b5f26 100644 --- a/public/app/features/admin/AdminFeatureTogglesAPI.ts +++ b/public/app/features/admin/AdminFeatureTogglesAPI.ts @@ -1,62 +1,74 @@ -import { createApi, BaseQueryFn } from '@reduxjs/toolkit/query/react'; -import { lastValueFrom } from 'rxjs'; - import { getBackendSrv } from '@grafana/runtime'; -type QueryArgs = { - url: string; - method?: string; - body?: { featureToggles: FeatureToggle[] }; -}; - -const backendSrvBaseQuery = - ({ baseUrl }: { baseUrl: string }): BaseQueryFn => - async ({ url, method = 'GET', body }) => { - try { - const { data } = await lastValueFrom( - getBackendSrv().fetch({ - url: baseUrl + url, - method, - data: body, - }) - ); - return { data }; - } catch (error) { - return { error }; - } - }; - -export const togglesApi = createApi({ - reducerPath: 'togglesApi', - baseQuery: backendSrvBaseQuery({ baseUrl: '/api' }), - endpoints: (builder) => ({ - getManagerState: builder.query({ - query: () => ({ url: '/featuremgmt/state' }), - }), - getFeatureToggles: builder.query({ - query: () => ({ url: '/featuremgmt' }), - }), - updateFeatureToggles: builder.mutation({ - query: (updatedToggles) => ({ - url: '/featuremgmt', - method: 'POST', - body: { featureToggles: updatedToggles }, - }), - }), - }), -}); - -type FeatureToggle = { +export type FeatureToggle = { name: string; description?: string; enabled: boolean; readOnly?: boolean; + hidden?: boolean; }; -type FeatureMgmtState = { +export type CurrentTogglesState = { restartRequired: boolean; allowEditing: boolean; + toggles: FeatureToggle[]; }; -export const { useGetManagerStateQuery, useGetFeatureTogglesQuery, useUpdateFeatureTogglesMutation } = togglesApi; -export type { FeatureToggle, FeatureMgmtState }; +interface ResolvedToggleState { + kind: 'ResolvedToggleState'; + restartRequired?: boolean; + allowEditing?: boolean; + toggles?: K8sToggleSpec[]; // not used in patch + enabled: { [key: string]: boolean }; +} + +interface K8sToggleSpec { + name: string; + description: string; + enabled: boolean; + writeable: boolean; + source: K8sToggleSource; +} + +interface K8sToggleSource { + namespace: string; + name: string; +} + +interface FeatureTogglesAPI { + getFeatureToggles(): Promise; + updateFeatureToggles(toggles: FeatureToggle[]): Promise; +} + +class K8sAPI implements FeatureTogglesAPI { + baseURL = '/apis/featuretoggle.grafana.app/v0alpha1'; + + async getFeatureToggles(): Promise { + const current = await getBackendSrv().get(this.baseURL + '/current'); + return { + restartRequired: Boolean(current.restartRequired), + allowEditing: Boolean(current.allowEditing), + toggles: current.toggles!.map((t) => ({ + name: t.name, + description: t.description!, + enabled: t.enabled, + readOnly: !Boolean(t.writeable), + hidden: false, // only return visible things + })), + }; + } + updateFeatureToggles(toggles: FeatureToggle[]): Promise { + const patchBody: ResolvedToggleState = { + kind: 'ResolvedToggleState', + enabled: {}, + }; + toggles.forEach((t) => { + patchBody.enabled[t.name] = t.enabled; + }); + return getBackendSrv().patch(this.baseURL + '/current', patchBody); + } +} + +export const getTogglesAPI = (): FeatureTogglesAPI => { + return new K8sAPI(); +}; diff --git a/public/app/features/admin/AdminFeatureTogglesPage.tsx b/public/app/features/admin/AdminFeatureTogglesPage.tsx index 7c7fde06c5f..4d231b644fd 100644 --- a/public/app/features/admin/AdminFeatureTogglesPage.tsx +++ b/public/app/features/admin/AdminFeatureTogglesPage.tsx @@ -1,26 +1,24 @@ import { css } from '@emotion/css'; import React, { useState } from 'react'; +import { useAsync } from 'react-use'; import { GrafanaTheme2 } from '@grafana/data'; import { useStyles2, Icon } from '@grafana/ui'; import { Page } from 'app/core/components/Page/Page'; -import { useGetFeatureTogglesQuery, useGetManagerStateQuery } from './AdminFeatureTogglesAPI'; +import { getTogglesAPI } from './AdminFeatureTogglesAPI'; import { AdminFeatureTogglesTable } from './AdminFeatureTogglesTable'; export default function AdminFeatureTogglesPage() { - const { data: featureToggles, isLoading, isError } = useGetFeatureTogglesQuery(); - const { data: featureMgmtState } = useGetManagerStateQuery(); + const [reload] = useState(1); + const togglesApi = getTogglesAPI(); + const featureState = useAsync(() => togglesApi.getFeatureToggles(), [reload]); const [updateSuccessful, setUpdateSuccessful] = useState(false); - const styles = useStyles2(getStyles); - const getErrorMessage = () => { - return 'Error fetching feature toggles'; - }; - const handleUpdateSuccess = () => { setUpdateSuccessful(true); + // setReload(reload+1); << would trigger updating the server state! }; const EditingAlert = () => { @@ -30,7 +28,7 @@ export default function AdminFeatureTogglesPage() {
- {featureMgmtState?.restartRequired || updateSuccessful + {featureState.value?.restartRequired || updateSuccessful ? 'A restart is pending for your Grafana instance to apply the latest feature toggle changes' : 'Saving feature toggle changes will prompt a restart of the instance, which may take a few minutes'} @@ -54,15 +52,16 @@ export default function AdminFeatureTogglesPage() { return ( - + <> - {isError && getErrorMessage()} - {isLoading && 'Fetching feature toggles'} - {featureMgmtState?.allowEditing && } - {featureToggles && ( + {featureState.error} + {featureState.loading && 'Fetching feature toggles'} + + {featureState.value?.restartRequired && } + {featureState.value && ( )} diff --git a/public/app/features/admin/AdminFeatureTogglesTable.tsx b/public/app/features/admin/AdminFeatureTogglesTable.tsx index d67a60c71eb..67697699af7 100644 --- a/public/app/features/admin/AdminFeatureTogglesTable.tsx +++ b/public/app/features/admin/AdminFeatureTogglesTable.tsx @@ -2,7 +2,7 @@ import React, { useState, useRef } from 'react'; import { Switch, InteractiveTable, Tooltip, type CellProps, Button, type SortByFn } from '@grafana/ui'; -import { type FeatureToggle, useUpdateFeatureTogglesMutation } from './AdminFeatureTogglesAPI'; +import { FeatureToggle, getTogglesAPI } from './AdminFeatureTogglesAPI'; interface Props { featureToggles: FeatureToggle[]; @@ -30,10 +30,12 @@ const sortByEnabled: SortByFn = (a, b) => { }; export function AdminFeatureTogglesTable({ featureToggles, allowEditing, onUpdateSuccess }: Props) { + // sort manually, doesn't look like it can be automatically done in the table + featureToggles.sort((a, b) => a.name.localeCompare(b.name)); const serverToggles = useRef(featureToggles); const [localToggles, setLocalToggles] = useState(featureToggles); - const [updateFeatureToggles] = useUpdateFeatureTogglesMutation(); const [isSaving, setIsSaving] = useState(false); + const togglesApi = getTogglesAPI(); const handleToggleChange = (toggle: FeatureToggle, newValue: boolean) => { const updatedToggle = { ...toggle, enabled: newValue }; @@ -47,12 +49,10 @@ export function AdminFeatureTogglesTable({ featureToggles, allowEditing, onUpdat setIsSaving(true); try { const modifiedToggles = getModifiedToggles(); - const resp = await updateFeatureToggles(modifiedToggles); - if (!('error' in resp)) { - // server toggles successfully updated - serverToggles.current = [...localToggles]; - onUpdateSuccess(); - } + await togglesApi.updateFeatureToggles(modifiedToggles); + // Pretend the values came from a new request + serverToggles.current = [...localToggles]; + onUpdateSuccess(); // should trigger a new get } finally { setIsSaving(false); } diff --git a/public/app/store/configureStore.ts b/public/app/store/configureStore.ts index c2c6cc89625..a7a462bf3a8 100644 --- a/public/app/store/configureStore.ts +++ b/public/app/store/configureStore.ts @@ -1,7 +1,6 @@ import { configureStore as reduxConfigureStore, createListenerMiddleware } from '@reduxjs/toolkit'; import { setupListeners } from '@reduxjs/toolkit/query'; -import { togglesApi } from 'app/features/admin/AdminFeatureTogglesAPI'; import { browseDashboardsAPI } from 'app/features/browse-dashboards/api/browseDashboardsAPI'; import { publicDashboardApi } from 'app/features/dashboard/api/publicDashboardApi'; import { StoreState } from 'app/types/store'; @@ -29,8 +28,7 @@ export function configureStore(initialState?: Partial) { listenerMiddleware.middleware, alertingApi.middleware, publicDashboardApi.middleware, - browseDashboardsAPI.middleware, - togglesApi.middleware + browseDashboardsAPI.middleware ), devTools: process.env.NODE_ENV !== 'production', preloadedState: { From 810d14d88faae97d8fa4a6aad821ae6d015fe193 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 1 Feb 2024 22:06:28 -0800 Subject: [PATCH 83/84] K8s: use --runtime-config={group}/{version}=true to enabled APIs (#81614) --- pkg/cmd/grafana/apiserver/cmd.go | 52 +++++++++++++++++++++++- pkg/cmd/grafana/apiserver/cmd_test.go | 22 ++++++++++ pkg/cmd/grafana/apiserver/server.go | 18 ++++---- pkg/registry/apis/datasource/register.go | 8 +++- 4 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 pkg/cmd/grafana/apiserver/cmd_test.go diff --git a/pkg/cmd/grafana/apiserver/cmd.go b/pkg/cmd/grafana/apiserver/cmd.go index d24829dba19..50f6ee979bf 100644 --- a/pkg/cmd/grafana/apiserver/cmd.go +++ b/pkg/cmd/grafana/apiserver/cmd.go @@ -1,7 +1,9 @@ package apiserver import ( + "fmt" "os" + "strings" "github.com/spf13/cobra" genericapiserver "k8s.io/apiserver/pkg/server" @@ -13,6 +15,7 @@ import ( func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}) *cobra.Command { devAcknowledgementNotice := "The apiserver command is in heavy development. The entire setup is subject to change without notice" + runtimeConfig := "" cmd := &cobra.Command{ Use: "apiserver [api group(s)]", @@ -21,8 +24,13 @@ func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{} devAcknowledgementNotice, Example: "grafana apiserver example.grafana.app", RunE: func(c *cobra.Command, args []string) error { + apis, err := readRuntimeConfig(runtimeConfig) + if err != nil { + return err + } + // Load each group from the args - if err := o.loadAPIGroupBuilders(args[1:]); err != nil { + if err := o.loadAPIGroupBuilders(apis); err != nil { return err } @@ -43,6 +51,8 @@ func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{} }, } + cmd.Flags().StringVar(&runtimeConfig, "runtime-config", "", "A set of key=value pairs that enable or disable built-in APIs.") + // Register standard k8s flags with the command line o.RecommendedOptions = options.NewRecommendedOptions( defaultEtcdPathPrefix, @@ -61,3 +71,43 @@ func RunCLI() int { return cli.Run(cmd) } + +type apiConfig struct { + group string + version string + enabled bool +} + +func (a apiConfig) String() string { + return fmt.Sprintf("%s/%s=%v", a.group, a.version, a.enabled) +} + +// Supported options are: +// +// /=true|false for a specific API group and version (e.g. dashboards.grafana.app/v0alpha1=true) +// api/all=true|false controls all API versions +// api/ga=true|false controls all API versions of the form v[0-9]+ +// api/beta=true|false controls all API versions of the form v[0-9]+beta[0-9]+ +// api/alpha=true|false controls all API versions of the form v[0-9]+alpha[0-9]+`) +// +// See: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ +func readRuntimeConfig(cfg string) ([]apiConfig, error) { + if cfg == "" { + return nil, fmt.Errorf("missing --runtime-config={apiservers}") + } + parts := strings.Split(cfg, ",") + apis := make([]apiConfig, len(parts)) + for i, part := range parts { + idx0 := strings.Index(part, "/") + idx1 := strings.LastIndex(part, "=") + if idx1 < idx0 || idx0 < 0 { + return nil, fmt.Errorf("expected values in the form: group/version=true") + } + apis[i] = apiConfig{ + group: part[:idx0], + version: part[idx0+1 : idx1], + enabled: part[idx1+1:] == "true", + } + } + return apis, nil +} diff --git a/pkg/cmd/grafana/apiserver/cmd_test.go b/pkg/cmd/grafana/apiserver/cmd_test.go new file mode 100644 index 00000000000..bf9b4b920c7 --- /dev/null +++ b/pkg/cmd/grafana/apiserver/cmd_test.go @@ -0,0 +1,22 @@ +package apiserver + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAdminAPIEndpoint(t *testing.T) { + out, err := readRuntimeConfig("all/all=true,dashboards.grafana.app/v0alpha1=false") + require.NoError(t, err) + require.Equal(t, []apiConfig{ + {group: "all", version: "all", enabled: true}, + {group: "dashboards.grafana.app", version: "v0alpha1", enabled: false}, + }, out) + require.Equal(t, "all/all=true", fmt.Sprintf("%v", out[0])) + + // Empty is an error + _, err = readRuntimeConfig("") + require.Error(t, err) +} diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index cd36da5abda..ed9c4df6946 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -47,11 +47,15 @@ func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions { } } -func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error { +func (o *APIServerOptions) loadAPIGroupBuilders(runtime []apiConfig) error { o.builders = []builder.APIGroupBuilder{} - for _, g := range args { - switch g { - // No dependencies for testing + for _, gv := range runtime { + if !gv.enabled { + return fmt.Errorf("disabling apis is not yet supported") + } + switch gv.group { + case "all": + return fmt.Errorf("managing all APIs is not yet supported") case "example.grafana.app": o.builders = append(o.builders, example.NewTestingAPIBuilder()) // Only works with testdata @@ -69,18 +73,18 @@ func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error { ), ) case "testdata.datasource.grafana.app": - ds, err := server.InitializeDataSourceAPIServer(g) + ds, err := server.InitializeDataSourceAPIServer(gv.group) if err != nil { return err } o.builders = append(o.builders, ds) default: - return fmt.Errorf("unknown group: %s", g) + return fmt.Errorf("unsupported runtime-config: %v", gv) } } if len(o.builders) < 1 { - return fmt.Errorf("expected group name(s) in the command line arguments") + return fmt.Errorf("no apis matched ") } // Install schemas diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index b5069a13cef..b16e9987d4a 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -210,7 +210,13 @@ func (b *DataSourceAPIBuilder) getPluginContext(ctx context.Context, uid string) } func (b *DataSourceAPIBuilder) GetOpenAPIDefinitions() openapi.GetOpenAPIDefinitions { - return v0alpha1.GetOpenAPIDefinitions + return func(ref openapi.ReferenceCallback) map[string]openapi.OpenAPIDefinition { + defs := query.GetOpenAPIDefinitions(ref) // required when running standalone + for k, v := range v0alpha1.GetOpenAPIDefinitions(ref) { + defs[k] = v + } + return defs + } } // Register additional routes with the server From 795eb4a8d8a299543d222ccedec44f8eac38e005 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 1 Feb 2024 22:40:11 -0800 Subject: [PATCH 84/84] K8s/Snapshots: Add dashboardsnapshot api group (#77667) --- pkg/api/api.go | 3 - pkg/api/dashboard_snapshot.go | 217 +------- pkg/api/dashboard_snapshot_test.go | 21 +- pkg/apis/dashboardsnapshot/v0alpha1/doc.go | 6 + .../dashboardsnapshot/v0alpha1/register.go | 25 + pkg/apis/dashboardsnapshot/v0alpha1/types.go | 136 +++++ .../v0alpha1/zz_generated.deepcopy.go | 271 +++++++++ .../v0alpha1/zz_generated.defaults.go | 19 + .../v0alpha1/zz_generated.openapi.go | 521 ++++++++++++++++++ ...enerated.openapi_violation_exceptions.list | 3 + pkg/registry/apis/apis.go | 2 + .../apis/dashboardsnapshot/conversions.go | 71 +++ .../apis/dashboardsnapshot/exporter.go | 130 +++++ .../apis/dashboardsnapshot/options_storage.go | 91 +++ .../apis/dashboardsnapshot/register.go | 345 ++++++++++++ .../apis/dashboardsnapshot/sql_storage.go | 147 +++++ .../apis/dashboardsnapshot/sub_body.go | 60 ++ pkg/registry/apis/wireset.go | 2 + .../dashboardsnapshots/database/database.go | 24 +- .../database/database_test.go | 28 +- pkg/services/dashboardsnapshots/models.go | 27 +- pkg/services/dashboardsnapshots/service.go | 211 +++++++ .../dashboardsnapshots/service/service.go | 2 +- .../service/service_test.go | 11 +- pkg/setting/setting.go | 10 +- .../apis/dashboardsnapshot/snapshots_test.go | 80 +++ public/api-enterprise-spec.json | 71 ++- public/api-merged.json | 71 ++- .../sharing/ShareSnapshotTab.tsx | 5 +- .../components/ShareModal/ShareSnapshot.tsx | 4 +- .../dashboard/services/SnapshotSrv.ts | 125 ++++- public/openapi3.json | 71 ++- 32 files changed, 2543 insertions(+), 267 deletions(-) create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/doc.go create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/register.go create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/types.go create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.deepcopy.go create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.defaults.go create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi.go create mode 100644 pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi_violation_exceptions.list create mode 100644 pkg/registry/apis/dashboardsnapshot/conversions.go create mode 100644 pkg/registry/apis/dashboardsnapshot/exporter.go create mode 100644 pkg/registry/apis/dashboardsnapshot/options_storage.go create mode 100644 pkg/registry/apis/dashboardsnapshot/register.go create mode 100644 pkg/registry/apis/dashboardsnapshot/sql_storage.go create mode 100644 pkg/registry/apis/dashboardsnapshot/sub_body.go create mode 100644 pkg/tests/apis/dashboardsnapshot/snapshots_test.go diff --git a/pkg/api/api.go b/pkg/api/api.go index e8886baf0bc..2a857fbf65c 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -31,7 +31,6 @@ package api import ( "github.com/grafana/grafana/pkg/api/routing" - "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/middleware/requestmeta" ac "github.com/grafana/grafana/pkg/services/accesscontrol" @@ -49,8 +48,6 @@ import ( "github.com/grafana/grafana/pkg/services/user" ) -var plog = log.New("api") - // registerRoutes registers all API HTTP routes. func (hs *HTTPServer) registerRoutes() { reqNoAuth := middleware.NoAuth() diff --git a/pkg/api/dashboard_snapshot.go b/pkg/api/dashboard_snapshot.go index 2b0c5e5471b..67f4df9c7a7 100644 --- a/pkg/api/dashboard_snapshot.go +++ b/pkg/api/dashboard_snapshot.go @@ -1,32 +1,22 @@ package api import ( - "bytes" - "encoding/json" "errors" - "fmt" "net/http" "time" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/components/simplejson" + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" "github.com/grafana/grafana/pkg/infra/metrics" - "github.com/grafana/grafana/pkg/services/auth/identity" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" "github.com/grafana/grafana/pkg/services/guardian" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) -var client = &http.Client{ - Timeout: time.Second * 5, - Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}, -} - // swagger:route GET /snapshot/shared-options snapshots getSharingOptions // // Get snapshot sharing settings. @@ -43,58 +33,6 @@ func (hs *HTTPServer) GetSharingOptions(c *contextmodel.ReqContext) { }) } -type CreateExternalSnapshotResponse struct { - Key string `json:"key"` - DeleteKey string `json:"deleteKey"` - Url string `json:"url"` - DeleteUrl string `json:"deleteUrl"` -} - -func createExternalDashboardSnapshot(cmd dashboardsnapshots.CreateDashboardSnapshotCommand, externalSnapshotUrl string) (*CreateExternalSnapshotResponse, error) { - var createSnapshotResponse CreateExternalSnapshotResponse - message := map[string]any{ - "name": cmd.Name, - "expires": cmd.Expires, - "dashboard": cmd.Dashboard, - "key": cmd.Key, - "deleteKey": cmd.DeleteKey, - } - - messageBytes, err := simplejson.NewFromAny(message).Encode() - if err != nil { - return nil, err - } - - resp, err := client.Post(externalSnapshotUrl+"/api/snapshots", "application/json", bytes.NewBuffer(messageBytes)) - if err != nil { - return nil, err - } - defer func() { - if err := resp.Body.Close(); err != nil { - plog.Warn("Failed to close response body", "err", err) - } - }() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("create external snapshot response status code %d", resp.StatusCode) - } - - if err := json.NewDecoder(resp.Body).Decode(&createSnapshotResponse); err != nil { - return nil, err - } - - return &createSnapshotResponse, nil -} - -func createOriginalDashboardURL(cmd *dashboardsnapshots.CreateDashboardSnapshotCommand) (string, error) { - dashUID := cmd.Dashboard.Get("uid").MustString("") - if ok := util.IsValidShortUID(dashUID); !ok { - return "", fmt.Errorf("invalid dashboard UID") - } - - return fmt.Sprintf("/d/%v", dashUID), nil -} - // swagger:route POST /snapshots snapshots createDashboardSnapshot // // When creating a snapshot using the API, you have to provide the full dashboard payload including the snapshot data. This endpoint is designed for the Grafana UI. @@ -106,95 +44,13 @@ func createOriginalDashboardURL(cmd *dashboardsnapshots.CreateDashboardSnapshotC // 401: unauthorisedError // 403: forbiddenError // 500: internalServerError -func (hs *HTTPServer) CreateDashboardSnapshot(c *contextmodel.ReqContext) response.Response { - if !hs.Cfg.SnapshotEnabled { - c.JsonApiErr(http.StatusForbidden, "Dashboard Snapshots are disabled", nil) - return nil - } - - cmd := dashboardsnapshots.CreateDashboardSnapshotCommand{} - if err := web.Bind(c.Req, &cmd); err != nil { - return response.Error(http.StatusBadRequest, "bad request data", err) - } - if cmd.Name == "" { - cmd.Name = "Unnamed snapshot" - } - - userID, err := identity.UserIdentifier(c.SignedInUser.GetNamespacedID()) - if err != nil { - return response.Error(http.StatusInternalServerError, - "Failed to create external snapshot", err) - } - - var snapshotUrl string - cmd.ExternalURL = "" - cmd.OrgID = c.SignedInUser.GetOrgID() - cmd.UserID = userID - originalDashboardURL, err := createOriginalDashboardURL(&cmd) - if err != nil { - return response.Error(http.StatusInternalServerError, "Invalid app URL", err) - } - - if cmd.External { - if !hs.Cfg.ExternalEnabled { - c.JsonApiErr(http.StatusForbidden, "External dashboard creation is disabled", nil) - return nil - } - - resp, err := createExternalDashboardSnapshot(cmd, hs.Cfg.ExternalSnapshotUrl) - if err != nil { - c.JsonApiErr(http.StatusInternalServerError, "Failed to create external snapshot", err) - return nil - } - - snapshotUrl = resp.Url - cmd.Key = resp.Key - cmd.DeleteKey = resp.DeleteKey - cmd.ExternalURL = resp.Url - cmd.ExternalDeleteURL = resp.DeleteUrl - cmd.Dashboard = simplejson.New() - - metrics.MApiDashboardSnapshotExternal.Inc() - } else { - cmd.Dashboard.SetPath([]string{"snapshot", "originalUrl"}, originalDashboardURL) - - if cmd.Key == "" { - var err error - cmd.Key, err = util.GetRandomString(32) - if err != nil { - c.JsonApiErr(http.StatusInternalServerError, "Could not generate random string", err) - return nil - } - } - - if cmd.DeleteKey == "" { - var err error - cmd.DeleteKey, err = util.GetRandomString(32) - if err != nil { - c.JsonApiErr(http.StatusInternalServerError, "Could not generate random string", err) - return nil - } - } - - snapshotUrl = setting.ToAbsUrl("dashboard/snapshot/" + cmd.Key) - - metrics.MApiDashboardSnapshotCreate.Inc() - } - - result, err := hs.dashboardsnapshotsService.CreateDashboardSnapshot(c.Req.Context(), &cmd) - if err != nil { - c.JsonApiErr(http.StatusInternalServerError, "Failed to create snapshot", err) - return nil - } - - c.JSON(http.StatusOK, util.DynMap{ - "key": cmd.Key, - "deleteKey": cmd.DeleteKey, - "url": snapshotUrl, - "deleteUrl": setting.ToAbsUrl("api/snapshots-delete/" + cmd.DeleteKey), - "id": result.ID, - }) - return nil +func (hs *HTTPServer) CreateDashboardSnapshot(c *contextmodel.ReqContext) { + dashboardsnapshots.CreateDashboardSnapshot(c, dashboardsnapshot.SnapshotSharingOptions{ + SnapshotsEnabled: hs.Cfg.SnapshotEnabled, + ExternalEnabled: hs.Cfg.ExternalEnabled, + ExternalSnapshotName: hs.Cfg.ExternalSnapshotName, + ExternalSnapshotURL: hs.Cfg.ExternalSnapshotUrl, + }, hs.dashboardsnapshotsService) } // GET /api/snapshots/:key @@ -247,38 +103,6 @@ func (hs *HTTPServer) GetDashboardSnapshot(c *contextmodel.ReqContext) response. return response.JSON(http.StatusOK, dto).SetHeader("Cache-Control", "public, max-age=3600") } -func deleteExternalDashboardSnapshot(externalUrl string) error { - resp, err := client.Get(externalUrl) - if err != nil { - return err - } - - defer func() { - if err := resp.Body.Close(); err != nil { - plog.Warn("Failed to close response body", "err", err) - } - }() - - if resp.StatusCode == 200 { - return nil - } - - // Gracefully ignore "snapshot not found" errors as they could have already - // been removed either via the cleanup script or by request. - if resp.StatusCode == 500 { - var respJson map[string]any - if err := json.NewDecoder(resp.Body).Decode(&respJson); err != nil { - return err - } - - if respJson["message"] == "Failed to get dashboard snapshot" { - return nil - } - } - - return fmt.Errorf("unexpected response when deleting external snapshot, status code: %d", resp.StatusCode) -} - // swagger:route GET /snapshots-delete/{deleteKey} snapshots deleteDashboardSnapshotByDeleteKey // // Delete Snapshot by deleteKey. @@ -302,28 +126,16 @@ func (hs *HTTPServer) DeleteDashboardSnapshotByDeleteKey(c *contextmodel.ReqCont return response.Error(404, "Snapshot not found", nil) } - query := &dashboardsnapshots.GetDashboardSnapshotQuery{DeleteKey: key} - queryResult, err := hs.dashboardsnapshotsService.GetDashboardSnapshot(c.Req.Context(), query) + err := dashboardsnapshots.DeleteWithKey(c.Req.Context(), key, hs.dashboardsnapshotsService) if err != nil { - return response.Err(err) - } - - if queryResult.External { - err := deleteExternalDashboardSnapshot(queryResult.ExternalDeleteURL) - if err != nil { - return response.Error(500, "Failed to delete external dashboard", err) + if errors.Is(err, dashboardsnapshots.ErrBaseNotFound) { + return response.Error(404, "Snapshot not found", err) } - } - - cmd := &dashboardsnapshots.DeleteDashboardSnapshotCommand{DeleteKey: queryResult.DeleteKey} - - if err := hs.dashboardsnapshotsService.DeleteDashboardSnapshot(c.Req.Context(), cmd); err != nil { return response.Error(500, "Failed to delete dashboard snapshot", err) } return response.JSON(http.StatusOK, util.DynMap{ "message": "Snapshot deleted. It might take an hour before it's cleared from any CDN caches.", - "id": queryResult.ID, }) } @@ -357,8 +169,13 @@ func (hs *HTTPServer) DeleteDashboardSnapshot(c *contextmodel.ReqContext) respon return response.Error(http.StatusNotFound, "Failed to get dashboard snapshot", nil) } + // TODO: enforce org ID same + // if queryResult.OrgID != c.OrgID { + // return response.Error(http.StatusUnauthorized, "OrgID mismatch", nil) + // } + if queryResult.External { - err := deleteExternalDashboardSnapshot(queryResult.ExternalDeleteURL) + err := dashboardsnapshots.DeleteExternalDashboardSnapshot(queryResult.ExternalDeleteURL) if err != nil { return response.Error(http.StatusInternalServerError, "Failed to delete external dashboard", err) } diff --git a/pkg/api/dashboard_snapshot_test.go b/pkg/api/dashboard_snapshot_test.go index d8023a9e83e..3fd0553610e 100644 --- a/pkg/api/dashboard_snapshot_test.go +++ b/pkg/api/dashboard_snapshot_test.go @@ -9,13 +9,14 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/web/webtest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/db/dbtest" @@ -148,12 +149,11 @@ func TestDashboardSnapshotAPIEndpoint_singleSnapshot(t *testing.T) { sc.handlerFunc = hs.DeleteDashboardSnapshotByDeleteKey sc.fakeReqWithParams("GET", sc.url, map[string]string{"deleteKey": "12345"}).exec() - require.Equal(t, 200, sc.resp.Code) + require.Equal(t, 200, sc.resp.Code, "BODY: "+sc.resp.Body.String()) respJSON, err := simplejson.NewJson(sc.resp.Body.Bytes()) require.NoError(t, err) assert.True(t, strings.HasPrefix(respJSON.Get("message").MustString(), "Snapshot deleted")) - assert.Equal(t, 1, respJSON.Get("id").MustInt()) assert.Equal(t, http.MethodGet, externalRequest.Method) assert.Equal(t, ts.URL, fmt.Sprintf("http://%s", externalRequest.Host)) @@ -271,7 +271,7 @@ func TestGetDashboardSnapshotNotFound(t *testing.T) { sc.handlerFunc = hs.DeleteDashboardSnapshot sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"key": "12345"}).exec() - assert.Equal(t, http.StatusNotFound, sc.resp.Code) + assert.Equal(t, http.StatusNotFound, sc.resp.Code, "BODY: "+sc.resp.Body.String()) }, sqlmock) loggedInUserScenarioWithRole(t, @@ -282,7 +282,7 @@ func TestGetDashboardSnapshotNotFound(t *testing.T) { sc.handlerFunc = hs.DeleteDashboardSnapshotByDeleteKey sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"deleteKey": "12345"}).exec() - assert.Equal(t, http.StatusNotFound, sc.resp.Code) + assert.Equal(t, http.StatusNotFound, sc.resp.Code, "BODY: "+sc.resp.Body.String()) }, sqlmock) } @@ -345,7 +345,7 @@ func TestGetDashboardSnapshotFailure(t *testing.T) { sc.handlerFunc = hs.DeleteDashboardSnapshot sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"key": "12345"}).exec() - assert.Equal(t, http.StatusForbidden, sc.resp.Code) + assert.Equal(t, http.StatusForbidden, sc.resp.Code, "BODY: "+sc.resp.Body.String()) }, sqlmock) loggedInUserScenarioWithRole(t, @@ -356,7 +356,7 @@ func TestGetDashboardSnapshotFailure(t *testing.T) { sc.handlerFunc = hs.DeleteDashboardSnapshotByDeleteKey sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"deleteKey": "12345"}).exec() - assert.Equal(t, http.StatusInternalServerError, sc.resp.Code) + assert.Equal(t, http.StatusInternalServerError, sc.resp.Code, "BODY: "+sc.resp.Body.String()) }, sqlmock) loggedInUserScenarioWithRole(t, @@ -367,7 +367,7 @@ func TestGetDashboardSnapshotFailure(t *testing.T) { sc.handlerFunc = hs.DeleteDashboardSnapshotByDeleteKey sc.fakeReqWithParams("DELETE", sc.url, map[string]string{"deleteKey": "12345"}).exec() - assert.Equal(t, http.StatusForbidden, sc.resp.Code) + assert.Equal(t, http.StatusForbidden, sc.resp.Code, "BODY: "+sc.resp.Body.String()) }, sqlmock) } @@ -391,6 +391,7 @@ func setUpSnapshotTest(t *testing.T, userId int64, deleteUrl string) dashboardsn res := &dashboardsnapshots.DashboardSnapshot{ ID: 1, + OrgID: 1, Key: "12345", DeleteKey: "54321", Dashboard: jsonModel, diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/doc.go b/pkg/apis/dashboardsnapshot/v0alpha1/doc.go new file mode 100644 index 00000000000..a6b2fec52cb --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/doc.go @@ -0,0 +1,6 @@ +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +k8s:defaulter-gen=TypeMeta +// +groupName=dashboardsnapshot.grafana.app + +package v0alpha1 diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/register.go b/pkg/apis/dashboardsnapshot/v0alpha1/register.go new file mode 100644 index 00000000000..e8022d55bf7 --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/register.go @@ -0,0 +1,25 @@ +package v0alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" +) + +const ( + GROUP = "dashboardsnapshot.grafana.app" + VERSION = "v0alpha1" + APIVERSION = GROUP + "/" + VERSION +) + +var DashboardSnapshotResourceInfo = common.NewResourceInfo(GROUP, VERSION, + "dashboardsnapshots", "dashboardsnapshot", "DashboardSnapshot", + func() runtime.Object { return &DashboardSnapshot{} }, + func() runtime.Object { return &DashboardSnapshotList{} }, +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: GROUP, Version: VERSION} +) diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/types.go b/pkg/apis/dashboardsnapshot/v0alpha1/types.go new file mode 100644 index 00000000000..6a0785bdedb --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/types.go @@ -0,0 +1,136 @@ +package v0alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DashboardSnapshot struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Snapshot summary info + Spec SnapshotInfo `json:"spec"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DashboardSnapshotList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []DashboardSnapshot `json:"items,omitempty"` +} + +type SnapshotInfo struct { + Title string `json:"title,omitempty"` + // Optionally auto-remove the snapshot at a future date + Expires int64 `json:"expires,omitempty"` + // When set to true, the snapshot exists in a remote server + External bool `json:"external,omitempty"` + // The external URL where the snapshot can be seen + ExternalURL string `json:"externalUrl,omitempty"` + // The URL that created the dashboard originally + OriginalUrl string `json:"originalUrl,omitempty"` + // Snapshot creation timestamp + Timestamp string `json:"timestamp,omitempty"` +} + +// This is returned from the POST command +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DashboardSnapshotWithDeleteKey struct { + DashboardSnapshot `json:",inline"` + + // The delete key is only returned when the item is created. It is not returned from a get request + DeleteKey string `json:"deleteKey,omitempty"` +} + +// This is the snapshot returned from the subresource +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type FullDashboardSnapshot struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Snapshot summary info + Info SnapshotInfo `json:"info"` + + // The raw dashboard (unstructured for now) + Dashboard common.Unstructured `json:"dashboard"` +} + +// Each tenant, may have different sharing options +// This is currently set using custom.ini, but multi-tenant support will need +// to be managed differently +type SnapshotSharingOptions struct { + SnapshotsEnabled bool `json:"snapshotEnabled"` + ExternalSnapshotURL string `json:"externalSnapshotURL,omitempty"` + ExternalSnapshotName string `json:"externalSnapshotName,omitempty"` + ExternalEnabled bool `json:"externalEnabled,omitempty"` +} + +// These are the values expected to be sent from an end user +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DashboardCreateCommand struct { + metav1.TypeMeta `json:",inline"` + + // Snapshot name + // required:false + Name string `json:"name"` + + // The complete dashboard model. + // required:true + Dashboard *common.Unstructured `json:"dashboard" binding:"Required"` + + // When the snapshot should expire in seconds in seconds. Default is never to expire. + // required:false + // default:0 + Expires int64 `json:"expires"` + + // these are passed when storing an external snapshot ref + // Save the snapshot on an external server rather than locally. + // required:false + // default: false + External bool `json:"external"` +} + +// The create response +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type DashboardCreateResponse struct { + metav1.TypeMeta `json:",inline"` + + // The unique key + Key string `json:"key"` + + // A unique key that will allow delete + DeleteKey string `json:"deleteKey"` + + // Absolute URL to show the dashboard + URL string `json:"url"` + + // URL that will delete the response + DeleteURL string `json:"deleteUrl"` +} + +// Represents an options object that must be named for each namespace/team/user +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type SharingOptions struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Show the options inline + Spec SnapshotSharingOptions `json:"spec"` +} + +// Represents a list of namespaced options +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type SharingOptionsList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []SharingOptions `json:"items,omitempty"` +} diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.deepcopy.go b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..51b5075f310 --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.deepcopy.go @@ -0,0 +1,271 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v0alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardCreateCommand) DeepCopyInto(out *DashboardCreateCommand) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Dashboard != nil { + in, out := &in.Dashboard, &out.Dashboard + *out = (*in).DeepCopy() + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardCreateCommand. +func (in *DashboardCreateCommand) DeepCopy() *DashboardCreateCommand { + if in == nil { + return nil + } + out := new(DashboardCreateCommand) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DashboardCreateCommand) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardCreateResponse) DeepCopyInto(out *DashboardCreateResponse) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardCreateResponse. +func (in *DashboardCreateResponse) DeepCopy() *DashboardCreateResponse { + if in == nil { + return nil + } + out := new(DashboardCreateResponse) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DashboardCreateResponse) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardSnapshot) DeepCopyInto(out *DashboardSnapshot) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardSnapshot. +func (in *DashboardSnapshot) DeepCopy() *DashboardSnapshot { + if in == nil { + return nil + } + out := new(DashboardSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DashboardSnapshot) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardSnapshotList) DeepCopyInto(out *DashboardSnapshotList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DashboardSnapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardSnapshotList. +func (in *DashboardSnapshotList) DeepCopy() *DashboardSnapshotList { + if in == nil { + return nil + } + out := new(DashboardSnapshotList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DashboardSnapshotList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardSnapshotWithDeleteKey) DeepCopyInto(out *DashboardSnapshotWithDeleteKey) { + *out = *in + in.DashboardSnapshot.DeepCopyInto(&out.DashboardSnapshot) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardSnapshotWithDeleteKey. +func (in *DashboardSnapshotWithDeleteKey) DeepCopy() *DashboardSnapshotWithDeleteKey { + if in == nil { + return nil + } + out := new(DashboardSnapshotWithDeleteKey) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DashboardSnapshotWithDeleteKey) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FullDashboardSnapshot) DeepCopyInto(out *FullDashboardSnapshot) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Info = in.Info + in.Dashboard.DeepCopyInto(&out.Dashboard) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FullDashboardSnapshot. +func (in *FullDashboardSnapshot) DeepCopy() *FullDashboardSnapshot { + if in == nil { + return nil + } + out := new(FullDashboardSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FullDashboardSnapshot) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SharingOptions) DeepCopyInto(out *SharingOptions) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharingOptions. +func (in *SharingOptions) DeepCopy() *SharingOptions { + if in == nil { + return nil + } + out := new(SharingOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SharingOptions) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SharingOptionsList) DeepCopyInto(out *SharingOptionsList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SharingOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharingOptionsList. +func (in *SharingOptionsList) DeepCopy() *SharingOptionsList { + if in == nil { + return nil + } + out := new(SharingOptionsList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SharingOptionsList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SnapshotInfo) DeepCopyInto(out *SnapshotInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotInfo. +func (in *SnapshotInfo) DeepCopy() *SnapshotInfo { + if in == nil { + return nil + } + out := new(SnapshotInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SnapshotSharingOptions) DeepCopyInto(out *SnapshotSharingOptions) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotSharingOptions. +func (in *SnapshotSharingOptions) DeepCopy() *SnapshotSharingOptions { + if in == nil { + return nil + } + out := new(SnapshotSharingOptions) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.defaults.go b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.defaults.go new file mode 100644 index 00000000000..238fc2f4edc --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.defaults.go @@ -0,0 +1,19 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v0alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi.go b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi.go new file mode 100644 index 00000000000..f5824a7d78a --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi.go @@ -0,0 +1,521 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// SPDX-License-Identifier: AGPL-3.0-only + +// Code generated by openapi-gen. DO NOT EDIT. + +// This file was autogenerated by openapi-gen. Do not edit it manually! + +package v0alpha1 + +import ( + common "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + return map[string]common.OpenAPIDefinition{ + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardCreateCommand": schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardCreateCommand(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardCreateResponse": schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardCreateResponse(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardSnapshot": schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardSnapshot(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardSnapshotList": schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardSnapshotList(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardSnapshotWithDeleteKey": schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardSnapshotWithDeleteKey(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.FullDashboardSnapshot": schema_pkg_apis_dashboardsnapshot_v0alpha1_FullDashboardSnapshot(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SharingOptions": schema_pkg_apis_dashboardsnapshot_v0alpha1_SharingOptions(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SharingOptionsList": schema_pkg_apis_dashboardsnapshot_v0alpha1_SharingOptionsList(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo": schema_pkg_apis_dashboardsnapshot_v0alpha1_SnapshotInfo(ref), + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotSharingOptions": schema_pkg_apis_dashboardsnapshot_v0alpha1_SnapshotSharingOptions(ref), + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardCreateCommand(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "These are the values expected to be sent from an end user", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Snapshot name required:false", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "dashboard": { + SchemaProps: spec.SchemaProps{ + Description: "The complete dashboard model. required:true", + Ref: ref("github.com/grafana/grafana/pkg/apis/common/v0alpha1.Unstructured"), + }, + }, + "expires": { + SchemaProps: spec.SchemaProps{ + Description: "When the snapshot should expire in seconds in seconds. Default is never to expire. required:false default:0", + Default: 0, + Type: []string{"integer"}, + Format: "int64", + }, + }, + "external": { + SchemaProps: spec.SchemaProps{ + Description: "these are passed when storing an external snapshot ref Save the snapshot on an external server rather than locally. required:false default: false", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"name", "dashboard", "expires", "external"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/common/v0alpha1.Unstructured"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardCreateResponse(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "The create response", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "key": { + SchemaProps: spec.SchemaProps{ + Description: "The unique key", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "deleteKey": { + SchemaProps: spec.SchemaProps{ + Description: "A unique key that will allow delete", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "url": { + SchemaProps: spec.SchemaProps{ + Description: "Absolute URL to show the dashboard", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "deleteUrl": { + SchemaProps: spec.SchemaProps{ + Description: "URL that will delete the response", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"key", "deleteKey", "url", "deleteUrl"}, + }, + }, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardSnapshot(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "Snapshot summary info", + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo"), + }, + }, + }, + Required: []string{"spec"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardSnapshotList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardSnapshot"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardSnapshot", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_DashboardSnapshotWithDeleteKey(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "This is returned from the POST command", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "Snapshot summary info", + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo"), + }, + }, + "deleteKey": { + SchemaProps: spec.SchemaProps{ + Description: "The delete key is only returned when the item is created. It is not returned from a get request", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"spec"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_FullDashboardSnapshot(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "This is the snapshot returned from the subresource", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "info": { + SchemaProps: spec.SchemaProps{ + Description: "Snapshot summary info", + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo"), + }, + }, + "dashboard": { + SchemaProps: spec.SchemaProps{ + Description: "The raw dashboard (unstructured for now)", + Ref: ref("github.com/grafana/grafana/pkg/apis/common/v0alpha1.Unstructured"), + }, + }, + }, + Required: []string{"info", "dashboard"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/common/v0alpha1.Unstructured", "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotInfo", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_SharingOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Represents an options object that must be named for each namespace/team/user", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "Show the options inline", + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotSharingOptions"), + }, + }, + }, + Required: []string{"spec"}, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SnapshotSharingOptions", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_SharingOptionsList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of namespaced options", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SharingOptions"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.SharingOptions", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"}, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_SnapshotInfo(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "title": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "expires": { + SchemaProps: spec.SchemaProps{ + Description: "Optionally auto-remove the snapshot at a future date", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "external": { + SchemaProps: spec.SchemaProps{ + Description: "When set to true, the snapshot exists in a remote server", + Type: []string{"boolean"}, + Format: "", + }, + }, + "externalUrl": { + SchemaProps: spec.SchemaProps{ + Description: "The external URL where the snapshot can be seen", + Type: []string{"string"}, + Format: "", + }, + }, + "originalUrl": { + SchemaProps: spec.SchemaProps{ + Description: "The URL that created the dashboard originally", + Type: []string{"string"}, + Format: "", + }, + }, + "timestamp": { + SchemaProps: spec.SchemaProps{ + Description: "Snapshot creation timestamp", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_dashboardsnapshot_v0alpha1_SnapshotSharingOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Each tenant, may have different sharing options This is currently set using custom.ini, but multi-tenant support will need to be managed differently", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "snapshotEnabled": { + SchemaProps: spec.SchemaProps{ + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + "externalSnapshotURL": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "externalSnapshotName": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "externalEnabled": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"snapshotEnabled"}, + }, + }, + } +} diff --git a/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi_violation_exceptions.list b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi_violation_exceptions.list new file mode 100644 index 00000000000..72af08dd20f --- /dev/null +++ b/pkg/apis/dashboardsnapshot/v0alpha1/zz_generated.openapi_violation_exceptions.list @@ -0,0 +1,3 @@ +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1,DashboardCreateResponse,DeleteURL +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1,SnapshotInfo,ExternalURL +API rule violation: names_match,github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1,SnapshotSharingOptions,SnapshotsEnabled \ No newline at end of file diff --git a/pkg/registry/apis/apis.go b/pkg/registry/apis/apis.go index 86f757eb8ba..2a918867edc 100644 --- a/pkg/registry/apis/apis.go +++ b/pkg/registry/apis/apis.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/registry/apis/dashboard" + "github.com/grafana/grafana/pkg/registry/apis/dashboardsnapshot" "github.com/grafana/grafana/pkg/registry/apis/datasource" "github.com/grafana/grafana/pkg/registry/apis/example" "github.com/grafana/grafana/pkg/registry/apis/featuretoggle" @@ -26,6 +27,7 @@ func ProvideRegistryServiceSink( _ *dashboard.DashboardsAPIBuilder, _ *playlist.PlaylistAPIBuilder, _ *example.TestingAPIBuilder, + _ *dashboardsnapshot.SnapshotsAPIBuilder, _ *featuretoggle.FeatureFlagAPIBuilder, _ *datasource.DataSourceAPIBuilder, _ *folders.FolderAPIBuilder, diff --git a/pkg/registry/apis/dashboardsnapshot/conversions.go b/pkg/registry/apis/dashboardsnapshot/conversions.go new file mode 100644 index 00000000000..8071799de5a --- /dev/null +++ b/pkg/registry/apis/dashboardsnapshot/conversions.go @@ -0,0 +1,71 @@ +package dashboardsnapshot + +import ( + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" +) + +func convertDTOToSnapshot(v *dashboardsnapshots.DashboardSnapshotDTO, namespacer request.NamespaceMapper) *dashboardsnapshot.DashboardSnapshot { + expires := v.Expires.UnixMilli() + if v.Expires.After(time.Date(2070, time.January, 0, 0, 0, 0, 0, time.UTC)) { + expires = 0 // ignore things expiring long into the future + } + snap := &dashboardsnapshot.DashboardSnapshot{ + TypeMeta: resourceInfo.TypeMeta(), + ObjectMeta: metav1.ObjectMeta{ + Name: v.Key, + ResourceVersion: fmt.Sprintf("%d", v.Updated.UnixMilli()), + CreationTimestamp: metav1.NewTime(v.Created), + Namespace: namespacer(v.OrgID), + }, + Spec: dashboardsnapshot.SnapshotInfo{ + Title: v.Name, + ExternalURL: v.ExternalURL, + Expires: expires, + }, + } + if v.Updated != v.Created { + meta, _ := utils.MetaAccessor(snap) + meta.SetUpdatedTimestamp(&v.Updated) + } + return snap +} + +func convertSnapshotToK8sResource(v *dashboardsnapshots.DashboardSnapshot, namespacer request.NamespaceMapper) *dashboardsnapshot.DashboardSnapshot { + expires := v.Expires.UnixMilli() + if v.Expires.After(time.Date(2070, time.January, 0, 0, 0, 0, 0, time.UTC)) { + expires = 0 // ignore things expiring long into the future + } + + info := dashboardsnapshot.SnapshotInfo{ + Title: v.Name, + ExternalURL: v.ExternalURL, + Expires: expires, + } + s := v.Dashboard.Get("snapshot") + if s != nil { + info.OriginalUrl, _ = s.Get("originalUrl").String() + info.Timestamp, _ = s.Get("timestamp").String() + } + snap := &dashboardsnapshot.DashboardSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: v.Key, + ResourceVersion: fmt.Sprintf("%d", v.Updated.UnixMilli()), + CreationTimestamp: metav1.NewTime(v.Created), + Namespace: namespacer(v.OrgID), + }, + Spec: info, + } + if v.Updated != v.Created { + meta, _ := utils.MetaAccessor(snap) + meta.SetUpdatedTimestamp(&v.Updated) + } + return snap +} diff --git a/pkg/registry/apis/dashboardsnapshot/exporter.go b/pkg/registry/apis/dashboardsnapshot/exporter.go new file mode 100644 index 00000000000..fdf8caab9e1 --- /dev/null +++ b/pkg/registry/apis/dashboardsnapshot/exporter.go @@ -0,0 +1,130 @@ +package dashboardsnapshot + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "gocloud.dev/blob" + "k8s.io/kube-openapi/pkg/spec3" + + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" +) + +type dashExportStatus struct { + Count int + Index int + Started int64 + Updated int64 + Finished int64 + Error string +} + +type dashExporter struct { + status dashExportStatus + + service dashboardsnapshots.Service + sql db.DB +} + +func (d *dashExporter) getAPIRouteHandler() builder.APIRouteHandler { + return builder.APIRouteHandler{ + Path: "admin/export", + Spec: &spec3.PathProps{ + Summary: "an example at the root level", + Description: "longer description here?", + Post: &spec3.Operation{ + OperationProps: spec3.OperationProps{ + Tags: []string{"export"}, + Responses: &spec3.Responses{ + ResponsesProps: spec3.ResponsesProps{ + StatusCodeResponses: map[int]*spec3.Response{ + 200: { + ResponseProps: spec3.ResponseProps{ + Content: map[string]*spec3.MediaType{ + "application/json": {}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Handler: func(w http.ResponseWriter, r *http.Request) { + // Only let it start once + if d.status.Started == 0 { + go d.doExport() + } + time.Sleep(time.Second) + _ = json.NewEncoder(w).Encode(d.status) + }, + } +} + +// NO way to stop!!!!!! +func (d *dashExporter) doExport() { + defer func() { + d.status.Finished = time.Now().UnixMilli() + }() + d.status = dashExportStatus{ + Started: time.Now().UnixMilli(), + } + if d.sql == nil { + d.status.Error = "missing dependencies" + return + } + + ctx := context.Background() + keys := []string{} + err := d.sql.GetSqlxSession().Select(ctx, + &keys, "SELECT key FROM dashboard_snapshot ORDER BY id asc") + if err != nil { + d.status.Error = err.Error() + return + } + d.status.Count = len(keys) + + bucket, err := blob.OpenBucket(ctx, "mem://?key=foo.txt&prefix=a/subfolder/") + if err != nil { + d.status.Error = err.Error() + return + } + defer func() { + _ = bucket.Close() + }() + + for idx, key := range keys { + d.status.Index = idx + snap, err := d.service.GetDashboardSnapshot(ctx, &dashboardsnapshots.GetDashboardSnapshotQuery{ + Key: key, + }) + if err != nil { + d.status.Error = err.Error() + return + } + + dash, err := snap.Dashboard.ToDB() + if err != nil { + d.status.Error = err.Error() + return + } + + fmt.Printf("TODO, export: %s (len: %d)\n", snap.Key, len(dash)) + + // w, err := bucket.NewWriter(ctx, "foo.txt", nil) + // if err != nil { + // d.status.Error = err.Error() + // return + // } + + time.Sleep(time.Second * 1) + d.status.Updated = time.Now().UnixMilli() + } + fmt.Printf("done!\n") +} diff --git a/pkg/registry/apis/dashboardsnapshot/options_storage.go b/pkg/registry/apis/dashboardsnapshot/options_storage.go new file mode 100644 index 00000000000..571010b9f5c --- /dev/null +++ b/pkg/registry/apis/dashboardsnapshot/options_storage.go @@ -0,0 +1,91 @@ +package dashboardsnapshot + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/setting" +) + +var ( + _ rest.Scoper = (*optionsStorage)(nil) + _ rest.SingularNameProvider = (*optionsStorage)(nil) + _ rest.Getter = (*optionsStorage)(nil) + _ rest.Lister = (*optionsStorage)(nil) + _ rest.Storage = (*optionsStorage)(nil) +) + +type sharingOptionsGetter = func(namespace string) (*dashboardsnapshot.SharingOptions, error) + +func newSharingOptionsGetter(cfg *setting.Cfg) sharingOptionsGetter { + s := &dashboardsnapshot.SharingOptions{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Now(), + }, + Spec: dashboardsnapshot.SnapshotSharingOptions{ + SnapshotsEnabled: cfg.SnapshotEnabled, + ExternalSnapshotURL: cfg.ExternalSnapshotUrl, + ExternalSnapshotName: cfg.ExternalSnapshotName, + ExternalEnabled: cfg.ExternalEnabled, + }, + } + return func(namespace string) (*dashboardsnapshot.SharingOptions, error) { + return s, nil + } +} + +type optionsStorage struct { + getter sharingOptionsGetter + tableConverter rest.TableConvertor +} + +func (s *optionsStorage) New() runtime.Object { + return &dashboardsnapshot.SharingOptions{} +} + +func (s *optionsStorage) Destroy() {} + +func (s *optionsStorage) NamespaceScoped() bool { + return true +} + +func (s *optionsStorage) GetSingularName() string { + return "options" +} + +func (s *optionsStorage) NewList() runtime.Object { + return &dashboardsnapshot.SharingOptionsList{} +} + +func (s *optionsStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) { + return s.tableConverter.ConvertToTable(ctx, object, tableOptions) +} + +func (s *optionsStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err != nil { + return nil, err + } + if info.OrgID < 0 { + return nil, fmt.Errorf("missing namespace") + } + v, err := s.getter(info.Value) + if err != nil { + return nil, err + } + list := &dashboardsnapshot.SharingOptionsList{ + Items: []dashboardsnapshot.SharingOptions{*v}, + } + return list, nil +} + +func (s *optionsStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { + return s.getter(name) +} diff --git a/pkg/registry/apis/dashboardsnapshot/register.go b/pkg/registry/apis/dashboardsnapshot/register.go new file mode 100644 index 00000000000..e59601696f3 --- /dev/null +++ b/pkg/registry/apis/dashboardsnapshot/register.go @@ -0,0 +1,345 @@ +package dashboardsnapshot + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/gorilla/mux" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/registry/generic" + "k8s.io/apiserver/pkg/registry/rest" + genericapiserver "k8s.io/apiserver/pkg/server" + common "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/spec3" + "k8s.io/kube-openapi/pkg/validation/spec" + + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" + "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/apiserver/builder" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/apiserver/utils" + contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" + "github.com/grafana/grafana/pkg/util/errutil/errhttp" + "github.com/grafana/grafana/pkg/web" +) + +var _ builder.APIGroupBuilder = (*SnapshotsAPIBuilder)(nil) + +var resourceInfo = dashboardsnapshot.DashboardSnapshotResourceInfo + +// This is used just so wire has something unique to return +type SnapshotsAPIBuilder struct { + service dashboardsnapshots.Service + namespacer request.NamespaceMapper + options sharingOptionsGetter + exporter *dashExporter + logger log.Logger +} + +func NewSnapshotsAPIBuilder( + p dashboardsnapshots.Service, + cfg *setting.Cfg, + exporter *dashExporter, +) *SnapshotsAPIBuilder { + return &SnapshotsAPIBuilder{ + service: p, + options: newSharingOptionsGetter(cfg), + namespacer: request.GetNamespaceMapper(cfg), + exporter: exporter, + logger: log.New("snapshots::RawHandlers"), + } +} + +func RegisterAPIService( + service dashboardsnapshots.Service, + apiregistration builder.APIRegistrar, + cfg *setting.Cfg, + features featuremgmt.FeatureToggles, + sql db.DB, +) *SnapshotsAPIBuilder { + if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { + return nil // skip registration unless opting into experimental apis + } + builder := NewSnapshotsAPIBuilder(service, cfg, &dashExporter{ + service: service, + sql: sql, + }) + apiregistration.RegisterAPI(builder) + return builder +} + +func (b *SnapshotsAPIBuilder) GetGroupVersion() schema.GroupVersion { + return resourceInfo.GroupVersion() +} + +func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { + scheme.AddKnownTypes(gv, + &dashboardsnapshot.DashboardSnapshot{}, + &dashboardsnapshot.DashboardSnapshotList{}, + &dashboardsnapshot.SharingOptions{}, + &dashboardsnapshot.SharingOptionsList{}, + &dashboardsnapshot.FullDashboardSnapshot{}, + &dashboardsnapshot.DashboardSnapshotWithDeleteKey{}, + &metav1.Status{}, + ) +} + +func (b *SnapshotsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { + gv := resourceInfo.GroupVersion() + addKnownTypes(scheme, gv) + + // Link this version to the internal representation. + // This is used for server-side-apply (PATCH), and avoids the error: + // "no kind is registered for the type" + addKnownTypes(scheme, schema.GroupVersion{ + Group: gv.Group, + Version: runtime.APIVersionInternal, + }) + + // If multiple versions exist, then register conversions from zz_generated.conversion.go + // if err := playlist.RegisterConversions(scheme); err != nil { + // return err + // } + metav1.AddToGroupVersion(scheme, gv) + return scheme.SetVersionPriority(gv) +} + +func (b *SnapshotsAPIBuilder) GetAPIGroupInfo( + scheme *runtime.Scheme, + codecs serializer.CodecFactory, // pointer? + optsGetter generic.RESTOptionsGetter, + dualWrite bool, +) (*genericapiserver.APIGroupInfo, error) { + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(dashboardsnapshot.GROUP, scheme, metav1.ParameterCodec, codecs) + storage := map[string]rest.Storage{} + + legacyStore := &legacyStorage{ + service: b.service, + namespacer: b.namespacer, + options: b.options, + } + legacyStore.tableConverter = utils.NewTableConverter( + resourceInfo.GroupResource(), + []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Title", Type: "string", Format: "string", Description: "The snapshot name"}, + {Name: "Created At", Type: "date"}, + }, + func(obj any) ([]interface{}, error) { + m, ok := obj.(*dashboardsnapshot.DashboardSnapshot) + if ok { + return []interface{}{ + m.Name, + m.Spec.Title, + m.CreationTimestamp.UTC().Format(time.RFC3339), + }, nil + } + return nil, fmt.Errorf("expected snapshot") + }, + ) + storage[resourceInfo.StoragePath()] = legacyStore + storage[resourceInfo.StoragePath("body")] = &subBodyREST{ + service: b.service, + namespacer: b.namespacer, + } + + storage["options"] = &optionsStorage{ + getter: b.options, + tableConverter: legacyStore.tableConverter, + } + + apiGroupInfo.VersionedResourcesStorageMap[dashboardsnapshot.VERSION] = storage + return &apiGroupInfo, nil +} + +func (b *SnapshotsAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions { + return dashboardsnapshot.GetOpenAPIDefinitions +} + +// Register additional routes with the server +func (b *SnapshotsAPIBuilder) GetAPIRoutes() *builder.APIRoutes { + prefix := dashboardsnapshot.DashboardSnapshotResourceInfo.GroupResource().Resource + defs := dashboardsnapshot.GetOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) + createCmd := defs["github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardCreateCommand"].Schema + createExample := `{"dashboard":{"annotations":{"list":[{"name":"Annotations & Alerts","enable":true,"iconColor":"rgba(0, 211, 255, 1)","snapshotData":[],"type":"dashboard","builtIn":1,"hide":true}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":203,"links":[],"liveNow":false,"panels":[{"datasource":null,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":43,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unitScale":true},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"id":1,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"10.4.0-pre","snapshotData":[{"fields":[{"config":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":43,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"showPoints":"auto","thresholdsStyle":{"mode":"off"}},"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unitScale":true},"name":"time","type":"time","values":[1706030536378,1706034856378,1706039176378,1706043496378,1706047816378,1706052136378]},{"config":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":43,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unitScale":true},"name":"A-series","type":"number","values":[1,20,90,30,50,0]}],"refId":"A"}],"targets":[],"title":"Simple example","type":"timeseries","links":[]}],"refresh":"","schemaVersion":39,"snapshot":{"timestamp":"2024-01-23T23:22:16.377Z"},"tags":[],"templating":{"list":[]},"time":{"from":"2024-01-23T17:22:20.380Z","to":"2024-01-23T23:22:20.380Z","raw":{"from":"now-6h","to":"now"}},"timepicker":{},"timezone":"","title":"simple and small","uid":"b22ec8db-399b-403b-b6c7-b0fb30ccb2a5","version":1,"weekStart":""},"name":"simple and small","expires":86400}` + createRsp := defs["github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1.DashboardCreateResponse"].Schema + + tags := []string{dashboardsnapshot.DashboardSnapshotResourceInfo.GroupVersionKind().Kind} + routes := &builder.APIRoutes{ + Namespace: []builder.APIRouteHandler{ + { + Path: prefix + "/create", + Spec: &spec3.PathProps{ + Summary: "an example at the root level", + Description: "longer description here?", + Post: &spec3.Operation{ + OperationProps: spec3.OperationProps{ + Tags: tags, + Parameters: []*spec3.Parameter{ + { + ParameterProps: spec3.ParameterProps{ + Name: "namespace", + In: "path", + Required: true, + Example: "default", + Description: "workspace", + Schema: spec.StringProperty(), + }, + }, + }, + RequestBody: &spec3.RequestBody{ + RequestBodyProps: spec3.RequestBodyProps{ + Content: map[string]*spec3.MediaType{ + "application/json": { + MediaTypeProps: spec3.MediaTypeProps{ + Schema: &createCmd, + Example: createExample, // raw JSON body + }, + }, + }, + }, + }, + Responses: &spec3.Responses{ + ResponsesProps: spec3.ResponsesProps{ + StatusCodeResponses: map[int]*spec3.Response{ + 200: { + ResponseProps: spec3.ResponseProps{ + Content: map[string]*spec3.MediaType{ + "application/json": { + MediaTypeProps: spec3.MediaTypeProps{ + Schema: &createRsp, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Handler: func(w http.ResponseWriter, r *http.Request) { + user, err := appcontext.User(r.Context()) + if err != nil { + errhttp.Write(r.Context(), err, w) + return + } + wrap := &contextmodel.ReqContext{ + Logger: b.logger, + Context: &web.Context{ + Req: r, + Resp: web.NewResponseWriter(r.Method, w), + }, + SignedInUser: user, + } + + vars := mux.Vars(r) + info, err := request.ParseNamespace(vars["namespace"]) + if err != nil { + wrap.JsonApiErr(http.StatusBadRequest, "expected namespace", nil) + return + } + if info.OrgID != user.OrgID { + wrap.JsonApiErr(http.StatusBadRequest, + fmt.Sprintf("user orgId does not match namespace (%d != %d)", info.OrgID, user.OrgID), nil) + return + } + opts, err := b.options(info.Value) + if err != nil { + wrap.JsonApiErr(http.StatusBadRequest, "error getting options", err) + return + } + + // Use the existing snapshot service + dashboardsnapshots.CreateDashboardSnapshot(wrap, opts.Spec, b.service) + }, + }, + { + Path: prefix + "/delete/{deleteKey}", + Spec: &spec3.PathProps{ + Summary: "an example at the root level", + Description: "longer description here?", + Delete: &spec3.Operation{ + OperationProps: spec3.OperationProps{ + Tags: tags, + Parameters: []*spec3.Parameter{ + { + ParameterProps: spec3.ParameterProps{ + Name: "deleteKey", + In: "path", + Required: true, + Description: "unique key returned in create", + Schema: spec.StringProperty(), + }, + }, + }, + }, + }, + }, + Handler: func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) + key := vars["deleteKey"] + + err := dashboardsnapshots.DeleteWithKey(ctx, key, b.service) + if err != nil { + errhttp.Write(ctx, fmt.Errorf("failed to delete external dashboard (%w)", err), w) + return + } + _ = json.NewEncoder(w).Encode(&util.DynMap{ + "message": "Snapshot deleted. It might take an hour before it's cleared from any CDN caches.", + }) + }, + }, + }, + } + + // dev environment to export all snapshots to a blob store + if b.exporter != nil && false { + routes.Root = append(routes.Root, b.exporter.getAPIRouteHandler()) + } + return routes +} + +func (b *SnapshotsAPIBuilder) GetAuthorizer() authorizer.Authorizer { + // TODO: this behavior must match the existing logic (it is currently more restrictive) + // + // https://github.com/grafana/grafana/blob/f63e43c113ac0cf8f78ed96ee2953874139bd2dc/pkg/middleware/auth.go#L203 + // func SnapshotPublicModeOrSignedIn(cfg *setting.Cfg) web.Handler { + // return func(c *contextmodel.ReqContext) { + // if cfg.SnapshotPublicMode { + // return + // } + + // if !c.IsSignedIn { + // notAuthorized(c) + // return + // } + // } + // } + + return authorizer.AuthorizerFunc( + func(ctx context.Context, attr authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + // Everyone can view dashsnaps + if attr.GetVerb() == "get" && attr.GetResource() == dashboardsnapshot.DashboardSnapshotResourceInfo.GroupResource().Resource { + return authorizer.DecisionAllow, "", err + } + + // Fallback to the default behaviors (namespace matches org) + return authorizer.DecisionNoOpinion, "", err + }) +} diff --git a/pkg/registry/apis/dashboardsnapshot/sql_storage.go b/pkg/registry/apis/dashboardsnapshot/sql_storage.go new file mode 100644 index 00000000000..b21231a8613 --- /dev/null +++ b/pkg/registry/apis/dashboardsnapshot/sql_storage.go @@ -0,0 +1,147 @@ +package dashboardsnapshot + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" + "github.com/grafana/grafana/pkg/infra/appcontext" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" +) + +var ( + _ rest.Scoper = (*legacyStorage)(nil) + _ rest.SingularNameProvider = (*legacyStorage)(nil) + _ rest.Getter = (*legacyStorage)(nil) + _ rest.Lister = (*legacyStorage)(nil) + _ rest.Storage = (*legacyStorage)(nil) + _ rest.GracefulDeleter = (*legacyStorage)(nil) +) + +type legacyStorage struct { + service dashboardsnapshots.Service + namespacer request.NamespaceMapper + tableConverter rest.TableConvertor + options sharingOptionsGetter +} + +func (s *legacyStorage) New() runtime.Object { + return resourceInfo.NewFunc() +} + +func (s *legacyStorage) Destroy() {} + +func (s *legacyStorage) NamespaceScoped() bool { + return true // namespace == org +} + +func (s *legacyStorage) GetSingularName() string { + return resourceInfo.GetSingularName() +} + +func (s *legacyStorage) NewList() runtime.Object { + return resourceInfo.NewListFunc() +} + +func (s *legacyStorage) checkEnabled(ns string) error { + opts, err := s.options(ns) + if err != nil { + return err + } + if !opts.Spec.SnapshotsEnabled { + return fmt.Errorf("snapshots not enabled") + } + return nil +} + +func (s *legacyStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) { + return s.tableConverter.ConvertToTable(ctx, object, tableOptions) +} + +func (s *legacyStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err == nil { + err = s.checkEnabled(info.Value) + } + if err != nil { + return nil, err + } + + user, err := appcontext.User(ctx) + if err != nil { + return nil, err + } + + limit := 5000 + if options.Limit > 0 { + limit = int(options.Limit) + } + res, err := s.service.SearchDashboardSnapshots(ctx, &dashboardsnapshots.GetDashboardSnapshotsQuery{ + OrgID: info.OrgID, + SignedInUser: user, + Limit: limit, + }) + if err != nil { + return nil, err + } + + list := &dashboardsnapshot.DashboardSnapshotList{} + for _, v := range res { + list.Items = append(list.Items, *convertDTOToSnapshot(v, s.namespacer)) + } + return list, nil +} + +func (s *legacyStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { + info, err := request.NamespaceInfoFrom(ctx, true) + if err == nil { + err = s.checkEnabled(info.Value) + } + if err != nil { + return nil, err + } + + v, err := s.service.GetDashboardSnapshot(ctx, &dashboardsnapshots.GetDashboardSnapshotQuery{ + Key: name, + }) + if err != nil || v == nil { + // if errors.Is(err, playlistsvc.ErrPlaylistNotFound) || err == nil { + // err = k8serrors.NewNotFound(s.SingularQualifiedResource, name) + // } + return nil, err + } + + return convertSnapshotToK8sResource(v, s.namespacer), nil +} + +// GracefulDeleter +func (s *legacyStorage) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { + snap, err := s.service.GetDashboardSnapshot(ctx, &dashboardsnapshots.GetDashboardSnapshotQuery{ + Key: name, + }) + if err != nil || snap == nil { + return nil, false, err + } + + // Delete the external one first + if snap.ExternalDeleteURL != "" { + err := dashboardsnapshots.DeleteExternalDashboardSnapshot(snap.ExternalDeleteURL) + if err != nil { + return nil, false, err + } + } + + err = s.service.DeleteDashboardSnapshot(ctx, &dashboardsnapshots.DeleteDashboardSnapshotCommand{ + DeleteKey: snap.DeleteKey, + }) + if err != nil { + return nil, false, err + } + return nil, true, nil +} diff --git a/pkg/registry/apis/dashboardsnapshot/sub_body.go b/pkg/registry/apis/dashboardsnapshot/sub_body.go new file mode 100644 index 00000000000..45614b5e635 --- /dev/null +++ b/pkg/registry/apis/dashboardsnapshot/sub_body.go @@ -0,0 +1,60 @@ +package dashboardsnapshot + +import ( + "context" + "net/http" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" + "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" +) + +type subBodyREST struct { + service dashboardsnapshots.Service + namespacer request.NamespaceMapper +} + +var _ = rest.Connecter(&subBodyREST{}) + +func (r *subBodyREST) New() runtime.Object { + return &dashboardsnapshot.FullDashboardSnapshot{} +} + +func (r *subBodyREST) Destroy() {} + +func (r *subBodyREST) ConnectMethods() []string { + return []string{"GET"} +} + +func (r *subBodyREST) NewConnectOptions() (runtime.Object, bool, string) { + return nil, false, "" +} + +func (r *subBodyREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + snap, err := r.service.GetDashboardSnapshot(ctx, &dashboardsnapshots.GetDashboardSnapshotQuery{ + Key: name, + }) + if err != nil { + responder.Error(err) + return + } + + data, err := snap.Dashboard.Map() + if err != nil { + responder.Error(err) + return + } + + r := convertSnapshotToK8sResource(snap, r.namespacer) + responder.Object(200, &dashboardsnapshot.FullDashboardSnapshot{ + ObjectMeta: r.ObjectMeta, + Info: r.Spec, + Dashboard: common.Unstructured{Object: data}, + }) + }), nil +} diff --git a/pkg/registry/apis/wireset.go b/pkg/registry/apis/wireset.go index e093f7a175e..3c5758da87b 100644 --- a/pkg/registry/apis/wireset.go +++ b/pkg/registry/apis/wireset.go @@ -4,6 +4,7 @@ import ( "github.com/google/wire" "github.com/grafana/grafana/pkg/registry/apis/dashboard" + "github.com/grafana/grafana/pkg/registry/apis/dashboardsnapshot" "github.com/grafana/grafana/pkg/registry/apis/datasource" "github.com/grafana/grafana/pkg/registry/apis/example" "github.com/grafana/grafana/pkg/registry/apis/featuretoggle" @@ -26,6 +27,7 @@ var WireSet = wire.NewSet( playlist.RegisterAPIService, dashboard.RegisterAPIService, example.RegisterAPIService, + dashboardsnapshot.RegisterAPIService, featuretoggle.RegisterAPIService, datasource.RegisterAPIService, folders.RegisterAPIService, diff --git a/pkg/services/dashboardsnapshots/database/database.go b/pkg/services/dashboardsnapshots/database/database.go index 4d1bb798df9..0acd68838bf 100644 --- a/pkg/services/dashboardsnapshots/database/database.go +++ b/pkg/services/dashboardsnapshots/database/database.go @@ -16,26 +16,36 @@ import ( type DashboardSnapshotStore struct { store db.DB log log.Logger - cfg *setting.Cfg + + // deprecated behavior + skipDeleteExpired bool } // DashboardStore implements the Store interface var _ dashboardsnapshots.Store = (*DashboardSnapshotStore)(nil) func ProvideStore(db db.DB, cfg *setting.Cfg) *DashboardSnapshotStore { - return &DashboardSnapshotStore{store: db, log: log.New("dashboardsnapshot.store"), cfg: cfg} + // nolint:staticcheck + return NewStore(db, !cfg.SnapShotRemoveExpired) +} + +func NewStore(db db.DB, skipDeleteExpired bool) *DashboardSnapshotStore { + log := log.New("dashboardsnapshot.store") + if skipDeleteExpired { + log.Warn("[Deprecated] The snapshot_remove_expired setting is outdated. Please remove from your config.") + } + return &DashboardSnapshotStore{store: db, skipDeleteExpired: skipDeleteExpired} } // DeleteExpiredSnapshots removes snapshots with old expiry dates. // SnapShotRemoveExpired is deprecated and should be removed in the future. // Snapshot expiry is decided by the user when they share the snapshot. func (d *DashboardSnapshotStore) DeleteExpiredSnapshots(ctx context.Context, cmd *dashboardsnapshots.DeleteExpiredSnapshotsCommand) error { + if d.skipDeleteExpired { + d.log.Warn("[Deprecated] The snapshot_remove_expired setting is outdated. Please remove from your config.") + return nil + } return d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error { - if !d.cfg.SnapShotRemoveExpired { - d.log.Warn("[Deprecated] The snapshot_remove_expired setting is outdated. Please remove from your config.") - return nil - } - deleteExpiredSQL := "DELETE FROM dashboard_snapshot WHERE expires < ?" expiredResponse, err := sess.Exec(deleteExpiredSQL, time.Now()) if err != nil { diff --git a/pkg/services/dashboardsnapshots/database/database_test.go b/pkg/services/dashboardsnapshots/database/database_test.go index 6982d241776..f19461d4fdb 100644 --- a/pkg/services/dashboardsnapshots/database/database_test.go +++ b/pkg/services/dashboardsnapshots/database/database_test.go @@ -8,6 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" @@ -116,9 +118,11 @@ func TestIntegrationDashboardSnapshotDBAccess(t *testing.T) { cmd := dashboardsnapshots.CreateDashboardSnapshotCommand{ Key: "strangesnapshotwithuserid0", DeleteKey: "adeletekey", - Dashboard: simplejson.NewFromAny(map[string]any{ - "hello": "mupp", - }), + DashboardCreateCommand: dashboardsnapshot.DashboardCreateCommand{ + Dashboard: &common.Unstructured{Object: map[string]any{ + "hello": "mupp", + }}, + }, UserID: 0, OrgID: 1, } @@ -155,11 +159,9 @@ func TestIntegrationDeleteExpiredSnapshots(t *testing.T) { t.Skip("skipping integration test") } sqlstore := db.InitTestDB(t) - dashStore := ProvideStore(sqlstore, setting.NewCfg()) + dashStore := NewStore(sqlstore, false) t.Run("Testing dashboard snapshots clean up", func(t *testing.T) { - dashStore.cfg.SnapShotRemoveExpired = true - nonExpiredSnapshot := createTestSnapshot(t, dashStore, "key1", 48000) createTestSnapshot(t, dashStore, "key2", -1200) createTestSnapshot(t, dashStore, "key3", -1200) @@ -196,12 +198,14 @@ func createTestSnapshot(t *testing.T, dashStore *DashboardSnapshotStore, key str cmd := dashboardsnapshots.CreateDashboardSnapshotCommand{ Key: key, DeleteKey: "delete" + key, - Dashboard: simplejson.NewFromAny(map[string]any{ - "hello": "mupp", - }), - UserID: 1000, - OrgID: 1, - Expires: expires, + DashboardCreateCommand: dashboardsnapshot.DashboardCreateCommand{ + Expires: expires, + Dashboard: &common.Unstructured{Object: map[string]any{ + "hello": "mupp", + }}, + }, + UserID: 1000, + OrgID: 1, } result, err := dashStore.CreateDashboardSnapshot(context.Background(), &cmd) require.NoError(t, err) diff --git a/pkg/services/dashboardsnapshots/models.go b/pkg/services/dashboardsnapshots/models.go index 36c155b52ea..7e4becec46b 100644 --- a/pkg/services/dashboardsnapshots/models.go +++ b/pkg/services/dashboardsnapshots/models.go @@ -3,6 +3,7 @@ package dashboardsnapshots import ( "time" + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/services/auth/identity" ) @@ -47,28 +48,17 @@ type DashboardSnapshotDTO struct { // swagger:model type CreateDashboardSnapshotCommand struct { - // The complete dashboard model. - // required:true - Dashboard *simplejson.Json `json:"dashboard" binding:"Required"` - // Snapshot name - // required:false - Name string `json:"name"` - // When the snapshot should expire in seconds in seconds. Default is never to expire. - // required:false - // default:0 - Expires int64 `json:"expires"` + // The "public" fields are defined in this struct while the private/SQL/response params are + // defied in the rest of this command + dashboardsnapshot.DashboardCreateCommand - // these are passed when storing an external snapshot ref - // Save the snapshot on an external server rather than locally. - // required:false - // default: false - External bool `json:"external"` ExternalURL string `json:"-"` ExternalDeleteURL string `json:"-"` // Define the unique key. Required if `external` is `true`. // required:false Key string `json:"key"` + // Unique key used to delete the snapshot. It is different from the `key` so that only the creator can delete the snapshot. Required if `external` is `true`. // required:false DeleteKey string `json:"deleteKey"` @@ -100,3 +90,10 @@ type GetDashboardSnapshotsQuery struct { OrgID int64 SignedInUser identity.Requester } + +type CreateExternalSnapshotResponse struct { + Key string `json:"key"` + DeleteKey string `json:"deleteKey"` + Url string `json:"url"` + DeleteUrl string `json:"deleteUrl"` +} diff --git a/pkg/services/dashboardsnapshots/service.go b/pkg/services/dashboardsnapshots/service.go index ae229d27fed..07bcea602c1 100644 --- a/pkg/services/dashboardsnapshots/service.go +++ b/pkg/services/dashboardsnapshots/service.go @@ -1,7 +1,23 @@ package dashboardsnapshots import ( + "bytes" "context" + "encoding/json" + "fmt" + "net/http" + "time" + + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/metrics" + "github.com/grafana/grafana/pkg/services/auth/identity" + contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" + "github.com/grafana/grafana/pkg/web" ) //go:generate mockery --name Service --structname MockService --inpackage --filename service_mock.go @@ -12,3 +28,198 @@ type Service interface { GetDashboardSnapshot(context.Context, *GetDashboardSnapshotQuery) (*DashboardSnapshot, error) SearchDashboardSnapshots(context.Context, *GetDashboardSnapshotsQuery) (DashboardSnapshotsList, error) } + +var client = &http.Client{ + Timeout: time.Second * 5, + Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}, +} + +func CreateDashboardSnapshot(c *contextmodel.ReqContext, cfg dashboardsnapshot.SnapshotSharingOptions, svc Service) { + if !cfg.SnapshotsEnabled { + c.JsonApiErr(http.StatusForbidden, "Dashboard Snapshots are disabled", nil) + return + } + + cmd := CreateDashboardSnapshotCommand{} + if err := web.Bind(c.Req, &cmd); err != nil { + c.JsonApiErr(http.StatusBadRequest, "bad request data", err) + return + } + if cmd.Name == "" { + cmd.Name = "Unnamed snapshot" + } + + userID, err := identity.UserIdentifier(c.SignedInUser.GetNamespacedID()) + if err != nil { + c.JsonApiErr(http.StatusInternalServerError, + "Failed to create external snapshot", err) + return + } + + var snapshotUrl string + cmd.ExternalURL = "" + cmd.OrgID = c.SignedInUser.GetOrgID() + cmd.UserID = userID + originalDashboardURL, err := createOriginalDashboardURL(&cmd) + if err != nil { + c.JsonApiErr(http.StatusInternalServerError, "Invalid app URL", err) + return + } + + if cmd.External { + if !cfg.ExternalEnabled { + c.JsonApiErr(http.StatusForbidden, "External dashboard creation is disabled", nil) + return + } + + resp, err := createExternalDashboardSnapshot(cmd, cfg.ExternalSnapshotURL) + if err != nil { + c.JsonApiErr(http.StatusInternalServerError, "Failed to create external snapshot", err) + return + } + + snapshotUrl = resp.Url + cmd.Key = resp.Key + cmd.DeleteKey = resp.DeleteKey + cmd.ExternalURL = resp.Url + cmd.ExternalDeleteURL = resp.DeleteUrl + cmd.Dashboard = &common.Unstructured{} + + metrics.MApiDashboardSnapshotExternal.Inc() + } else { + cmd.Dashboard.SetNestedField(originalDashboardURL, "snapshot", "originalUrl") + + if cmd.Key == "" { + var err error + cmd.Key, err = util.GetRandomString(32) + if err != nil { + c.JsonApiErr(http.StatusInternalServerError, "Could not generate random string", err) + return + } + } + + if cmd.DeleteKey == "" { + var err error + cmd.DeleteKey, err = util.GetRandomString(32) + if err != nil { + c.JsonApiErr(http.StatusInternalServerError, "Could not generate random string", err) + return + } + } + + snapshotUrl = setting.ToAbsUrl("dashboard/snapshot/" + cmd.Key) + + metrics.MApiDashboardSnapshotCreate.Inc() + } + + result, err := svc.CreateDashboardSnapshot(c.Req.Context(), &cmd) + if err != nil { + c.JsonApiErr(http.StatusInternalServerError, "Failed to create snapshot", err) + return + } + + c.JSON(http.StatusOK, dashboardsnapshot.DashboardCreateResponse{ + Key: result.Key, + DeleteKey: result.DeleteKey, + URL: snapshotUrl, + DeleteURL: setting.ToAbsUrl("api/snapshots-delete/" + result.DeleteKey), + }) +} + +var plog = log.New("external-snapshot") + +func DeleteExternalDashboardSnapshot(externalUrl string) error { + resp, err := client.Get(externalUrl) + if err != nil { + return err + } + + defer func() { + if err := resp.Body.Close(); err != nil { + plog.Warn("Failed to close response body", "err", err) + } + }() + + if resp.StatusCode == 200 { + return nil + } + + // Gracefully ignore "snapshot not found" errors as they could have already + // been removed either via the cleanup script or by request. + if resp.StatusCode == 500 { + var respJson map[string]any + if err := json.NewDecoder(resp.Body).Decode(&respJson); err != nil { + return err + } + + if respJson["message"] == "Failed to get dashboard snapshot" { + return nil + } + } + + return fmt.Errorf("unexpected response when deleting external snapshot, status code: %d", resp.StatusCode) +} + +func createExternalDashboardSnapshot(cmd CreateDashboardSnapshotCommand, externalSnapshotUrl string) (*CreateExternalSnapshotResponse, error) { + var createSnapshotResponse CreateExternalSnapshotResponse + message := map[string]any{ + "name": cmd.Name, + "expires": cmd.Expires, + "dashboard": cmd.Dashboard, + "key": cmd.Key, + "deleteKey": cmd.DeleteKey, + } + + messageBytes, err := simplejson.NewFromAny(message).Encode() + if err != nil { + return nil, err + } + + resp, err := client.Post(externalSnapshotUrl+"/api/snapshots", "application/json", bytes.NewBuffer(messageBytes)) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + plog.Warn("Failed to close response body", "err", err) + } + }() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("create external snapshot response status code %d", resp.StatusCode) + } + + if err := json.NewDecoder(resp.Body).Decode(&createSnapshotResponse); err != nil { + return nil, err + } + + return &createSnapshotResponse, nil +} + +func createOriginalDashboardURL(cmd *CreateDashboardSnapshotCommand) (string, error) { + dashUID := cmd.Dashboard.GetNestedString("uid") + if ok := util.IsValidShortUID(dashUID); !ok { + return "", fmt.Errorf("invalid dashboard UID") + } + + return fmt.Sprintf("/d/%v", dashUID), nil +} + +func DeleteWithKey(ctx context.Context, key string, svc Service) error { + query := &GetDashboardSnapshotQuery{DeleteKey: key} + queryResult, err := svc.GetDashboardSnapshot(ctx, query) + if err != nil { + return err + } + + if queryResult.External { + err := DeleteExternalDashboardSnapshot(queryResult.ExternalDeleteURL) + if err != nil { + return err + } + } + + cmd := &DeleteDashboardSnapshotCommand{DeleteKey: queryResult.DeleteKey} + + return svc.DeleteDashboardSnapshot(ctx, cmd) +} diff --git a/pkg/services/dashboardsnapshots/service/service.go b/pkg/services/dashboardsnapshots/service/service.go index 247f0e7aec9..c26f00e7d23 100644 --- a/pkg/services/dashboardsnapshots/service/service.go +++ b/pkg/services/dashboardsnapshots/service/service.go @@ -26,7 +26,7 @@ func ProvideService(store dashboardsnapshots.Store, secretsService secrets.Servi } func (s *ServiceImpl) CreateDashboardSnapshot(ctx context.Context, cmd *dashboardsnapshots.CreateDashboardSnapshotCommand) (*dashboardsnapshots.DashboardSnapshot, error) { - marshalledData, err := cmd.Dashboard.Encode() + marshalledData, err := cmd.Dashboard.MarshalJSON() if err != nil { return nil, err } diff --git a/pkg/services/dashboardsnapshots/service/service_test.go b/pkg/services/dashboardsnapshots/service/service_test.go index 638a584d910..f4b80d9d9c9 100644 --- a/pkg/services/dashboardsnapshots/service/service_test.go +++ b/pkg/services/dashboardsnapshots/service/service_test.go @@ -2,11 +2,13 @@ package service import ( "context" + "encoding/json" "testing" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/components/simplejson" + common "github.com/grafana/grafana/pkg/apis/common/v0alpha1" + dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/dashboardsnapshots" dashsnapdb "github.com/grafana/grafana/pkg/services/dashboardsnapshots/database" @@ -30,8 +32,9 @@ func TestDashboardSnapshotsService(t *testing.T) { dashboardKey := "12345" + dashboard := &common.Unstructured{} rawDashboard := []byte(`{"id":123}`) - dashboard, err := simplejson.NewJson(rawDashboard) + err := json.Unmarshal(rawDashboard, dashboard) require.NoError(t, err) t.Run("create dashboard snapshot should encrypt the dashboard", func(t *testing.T) { @@ -40,7 +43,9 @@ func TestDashboardSnapshotsService(t *testing.T) { cmd := dashboardsnapshots.CreateDashboardSnapshotCommand{ Key: dashboardKey, DeleteKey: dashboardKey, - Dashboard: dashboard, + DashboardCreateCommand: dashboardsnapshot.DashboardCreateCommand{ + Dashboard: dashboard, + }, } result, err := s.CreateDashboardSnapshot(ctx, &cmd) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 8fdda78a12b..7cb19b47cfd 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -359,12 +359,14 @@ type Cfg struct { SqlDatasourceMaxConnLifetimeDefault int // Snapshots - SnapshotEnabled bool - ExternalSnapshotUrl string - ExternalSnapshotName string - ExternalEnabled bool + SnapshotEnabled bool + ExternalSnapshotUrl string + ExternalSnapshotName string + ExternalEnabled bool + // Deprecated: setting this to false adds deprecation warnings at runtime SnapShotRemoveExpired bool + // Only used in https://snapshots.raintank.io/ SnapshotPublicMode bool ErrTemplateName string diff --git a/pkg/tests/apis/dashboardsnapshot/snapshots_test.go b/pkg/tests/apis/dashboardsnapshot/snapshots_test.go new file mode 100644 index 00000000000..c25c4029d0a --- /dev/null +++ b/pkg/tests/apis/dashboardsnapshot/snapshots_test.go @@ -0,0 +1,80 @@ +package dashboardsnapshots + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/tests/apis" + "github.com/grafana/grafana/pkg/tests/testinfra" +) + +func TestDashboardSnapshots(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + AppModeProduction: false, // required for experimental apis + DisableAnonymous: true, + EnableFeatureToggles: []string{ + featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // required to register dashboardsnapshot.grafana.app + }, + }) + + t.Run("Check discovery client", func(t *testing.T) { + disco := helper.GetGroupVersionInfoJSON("dashboardsnapshot.grafana.app") + + // fmt.Printf("%s", disco) + require.JSONEq(t, `[ + { + "freshness": "Current", + "resources": [ + { + "resource": "dashboardsnapshot", + "responseKind": { + "group": "", + "kind": "DashboardSnapshot", + "version": "" + }, + "scope": "Namespaced", + "singularResource": "dashsnap", + "subresources": [ + { + "responseKind": { + "group": "", + "kind": "FullDashboardSnapshot", + "version": "" + }, + "subresource": "body", + "verbs": [ + "get" + ] + } + ], + "verbs": [ + "delete", + "get", + "list" + ] + }, + { + "resource": "options", + "responseKind": { + "group": "", + "kind": "SharingOptions", + "version": "" + }, + "scope": "Namespaced", + "singularResource": "options", + "verbs": [ + "get", + "list" + ] + } + ], + "version": "v0alpha1" + } + ]`, disco) + }) +} diff --git a/public/api-enterprise-spec.json b/public/api-enterprise-spec.json index 99da38d47d5..1fce6c5f1c1 100644 --- a/public/api-enterprise-spec.json +++ b/public/api-enterprise-spec.json @@ -3258,8 +3258,12 @@ "dashboard" ], "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, "dashboard": { - "$ref": "#/definitions/Json" + "$ref": "#/definitions/Unstructured" }, "deleteKey": { "description": "Unique key used to delete the snapshot. It is different from the `key` so that only the creator can delete the snapshot. Required if `external` is `true`.", @@ -3280,6 +3284,10 @@ "description": "Define the unique key. Required if `external` is `true`.", "type": "string" }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + }, "name": { "description": "Snapshot name", "type": "string" @@ -3618,6 +3626,41 @@ } } }, + "DashboardCreateCommand": { + "description": "These are the values expected to be sent from an end user\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", + "type": "object", + "required": [ + "dashboard" + ], + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, + "dashboard": { + "$ref": "#/definitions/Unstructured" + }, + "expires": { + "description": "When the snapshot should expire in seconds in seconds. Default is never to expire.", + "type": "integer", + "format": "int64", + "default": 0 + }, + "external": { + "description": "these are passed when storing an external snapshot ref\nSave the snapshot on an external server rather than locally.", + "type": "boolean", + "default": false + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + }, + "name": { + "description": "Snapshot name", + "type": "string" + } + } + }, "DashboardFullWithMeta": { "type": "object", "properties": { @@ -7380,6 +7423,21 @@ "Type": { "type": "string" }, + "TypeMeta": { + "description": "+k8s:deepcopy-gen=false", + "type": "object", + "title": "TypeMeta describes an individual object in an API response or request\nwith strings representing the type of the object and its API schema version.\nStructures that are versioned or persisted should inline TypeMeta.", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + } + } + }, "URL": { "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use the EscapedPath method, which preserves\nthe original encoding of Path.\n\nThe RawPath field is an optional field which is only set when the default\nencoding of Path is different from the escaped path. See the EscapedPath method\nfor more details.\n\nURL's String method uses the EscapedPath method to obtain the path.", "type": "object", @@ -7420,6 +7478,17 @@ } } }, + "Unstructured": { + "description": "Unstructured allows objects that do not have Golang structs registered to be manipulated\ngenerically.", + "type": "object", + "properties": { + "Object": { + "description": "Object is a JSON compatible map with string, float, int, bool, []interface{},\nor map[string]interface{} children.", + "type": "object", + "additionalProperties": {} + } + } + }, "UpdateAlertNotificationCommand": { "type": "object", "properties": { diff --git a/public/api-merged.json b/public/api-merged.json index 5828a0d7816..0595858477e 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -13578,8 +13578,12 @@ "dashboard" ], "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, "dashboard": { - "$ref": "#/definitions/Json" + "$ref": "#/definitions/Unstructured" }, "deleteKey": { "description": "Unique key used to delete the snapshot. It is different from the `key` so that only the creator can delete the snapshot. Required if `external` is `true`.", @@ -13600,6 +13604,10 @@ "description": "Define the unique key. Required if `external` is `true`.", "type": "string" }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + }, "name": { "description": "Snapshot name", "type": "string" @@ -13938,6 +13946,41 @@ } } }, + "DashboardCreateCommand": { + "description": "These are the values expected to be sent from an end user\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", + "type": "object", + "required": [ + "dashboard" + ], + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, + "dashboard": { + "$ref": "#/definitions/Unstructured" + }, + "expires": { + "description": "When the snapshot should expire in seconds in seconds. Default is never to expire.", + "type": "integer", + "format": "int64", + "default": 0 + }, + "external": { + "description": "these are passed when storing an external snapshot ref\nSave the snapshot on an external server rather than locally.", + "type": "boolean", + "default": false + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + }, + "name": { + "description": "Snapshot name", + "type": "string" + } + } + }, "DashboardFullWithMeta": { "type": "object", "properties": { @@ -20764,6 +20807,21 @@ "Type": { "type": "string" }, + "TypeMeta": { + "description": "+k8s:deepcopy-gen=false", + "type": "object", + "title": "TypeMeta describes an individual object in an API response or request\nwith strings representing the type of the object and its API schema version.\nStructures that are versioned or persisted should inline TypeMeta.", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + } + } + }, "URL": { "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use the EscapedPath method, which preserves\nthe original encoding of Path.\n\nThe RawPath field is an optional field which is only set when the default\nencoding of Path is different from the escaped path. See the EscapedPath method\nfor more details.\n\nURL's String method uses the EscapedPath method to obtain the path.", "type": "object", @@ -20804,6 +20862,17 @@ } } }, + "Unstructured": { + "description": "Unstructured allows objects that do not have Golang structs registered to be manipulated\ngenerically.", + "type": "object", + "properties": { + "Object": { + "description": "Object is a JSON compatible map with string, float, int, bool, []interface{},\nor map[string]interface{} children.", + "type": "object", + "additionalProperties": false + } + } + }, "UpdateAlertNotificationCommand": { "type": "object", "properties": { diff --git a/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx b/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx index c65584657f9..2e9503c3a11 100644 --- a/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx +++ b/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx @@ -15,8 +15,6 @@ import { DashboardInteractions } from '../utils/interactions'; import { SceneShareTabState } from './types'; -const SNAPSHOTS_API_ENDPOINT = '/api/snapshots'; - const getExpireOptions = () => { const DEFAULT_EXPIRE_OPTION: SelectableValue = { label: t('share-modal.snapshot.expire-never', `Never`), @@ -121,8 +119,7 @@ export class ShareSnapshotTab extends SceneObjectBase { }; try { - const results: { deleteUrl: string; url: string } = await getBackendSrv().post(SNAPSHOTS_API_ENDPOINT, cmdData); - return results; + return await getDashboardSnapshotSrv().create(cmdData); } finally { if (external) { DashboardInteractions.publishSnapshotClicked({ expires: cmdData.expires }); diff --git a/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx b/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx index 18130b09b57..2c93b86df28 100644 --- a/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx +++ b/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx @@ -13,8 +13,6 @@ import { getDashboardSnapshotSrv } from '../../services/SnapshotSrv'; import { ShareModalTabProps } from './types'; -const snapshotApiUrl = '/api/snapshots'; - interface Props extends ShareModalTabProps {} interface State { @@ -109,7 +107,7 @@ export class ShareSnapshot extends PureComponent { }; try { - const results: { deleteUrl: string; url: string } = await getBackendSrv().post(snapshotApiUrl, cmdData); + const results = await getDashboardSnapshotSrv().create(cmdData); this.setState({ deleteUrl: results.deleteUrl, snapshotUrl: results.url, diff --git a/public/app/features/dashboard/services/SnapshotSrv.ts b/public/app/features/dashboard/services/SnapshotSrv.ts index 293d5245fa3..e3e0e9db637 100644 --- a/public/app/features/dashboard/services/SnapshotSrv.ts +++ b/public/app/features/dashboard/services/SnapshotSrv.ts @@ -1,5 +1,8 @@ -import { getBackendSrv } from '@grafana/runtime'; -import { DashboardDTO } from 'app/types'; +import { lastValueFrom, map } from 'rxjs'; + +import { config, getBackendSrv, FetchResponse } from '@grafana/runtime'; +import { contextSrv } from 'app/core/core'; +import { DashboardDataDTO, DashboardDTO } from 'app/types'; // Used in the snapshot list export interface Snapshot { @@ -17,7 +20,21 @@ export interface SnapshotSharingOptions { snapshotEnabled: boolean; } +export interface SnapshotCreateCommand { + dashboard: object; + name: string; + expires?: number; + external?: boolean; +} + +export interface SnapshotCreateResponse { + key: string; + url: string; + deleteUrl: string; +} + export interface DashboardSnapshotSrv { + create: (cmd: SnapshotCreateCommand) => Promise; getSnapshots: () => Promise; getSharingOptions: () => Promise; deleteSnapshot: (key: string) => Promise; @@ -25,6 +42,7 @@ export interface DashboardSnapshotSrv { } const legacyDashboardSnapshotSrv: DashboardSnapshotSrv = { + create: (cmd: SnapshotCreateCommand) => getBackendSrv().post('/api/snapshots', cmd), getSnapshots: () => getBackendSrv().get('/api/dashboard/snapshots'), getSharingOptions: () => getBackendSrv().get('/api/snapshot/shared-options'), deleteSnapshot: (key: string) => getBackendSrv().delete('/api/snapshots/' + key), @@ -35,6 +53,109 @@ const legacyDashboardSnapshotSrv: DashboardSnapshotSrv = { }, }; +interface K8sMetadata { + name: string; + namespace: string; + resourceVersion: string; + creationTimestamp: string; +} + +interface K8sSnapshotInfo { + title: string; + externalUrl?: string; + expires?: number; +} + +interface K8sSnapshotResource { + metadata: K8sMetadata; + spec: K8sSnapshotInfo; +} + +interface DashboardSnapshotList { + items: K8sSnapshotResource[]; +} + +interface K8sDashboardSnapshot { + apiVersion: string; + kind: 'DashboardSnapshot'; + metadata: K8sMetadata; + dashboard: DashboardDataDTO; +} + +class K8sAPI implements DashboardSnapshotSrv { + readonly apiVersion = 'dashboardsnapshot.grafana.app/v0alpha1'; + readonly url: string; + + constructor() { + this.url = `/apis/${this.apiVersion}/namespaces/${config.namespace}/dashboardsnapshots`; + } + + async create(cmd: SnapshotCreateCommand) { + return getBackendSrv().post(this.url + '/create', cmd); + } + + async getSnapshots(): Promise { + const result = await getBackendSrv().get(this.url); + return result.items.map((r) => { + return { + key: r.metadata.name, + name: r.spec.title, + external: r.spec.externalUrl != null, + externalUrl: r.spec.externalUrl, + }; + }); + } + + deleteSnapshot(uid: string) { + return getBackendSrv().delete(this.url + '/' + uid); + } + + async getSharingOptions() { + // TODO? should this be in a config service, or in the same service? + // we have http://localhost:3000/apis/dashboardsnapshot.grafana.app/v0alpha1/namespaces/default/options + // BUT that has an unclear user mapping story still, so lets stick with the existing shared-options endpoint + return getBackendSrv().get('/api/snapshot/shared-options'); + } + + async getSnapshot(uid: string): Promise { + const headers: Record = {}; + if (!contextSrv.isSignedIn) { + alert('TODO... need a barer token for anonymous use case'); + const token = `??? TODO, get anon token for snapshots (${contextSrv.user?.name}) ???`; + headers['Authorization'] = `Bearer ${token}`; + } + return lastValueFrom( + getBackendSrv() + .fetch({ + url: this.url + '/' + uid + '/body', + method: 'GET', + headers: headers, + }) + .pipe( + map((response: FetchResponse) => { + return { + dashboard: response.data.dashboard, + meta: { + isSnapshot: true, + canSave: false, + canEdit: false, + canAdmin: false, + canStar: false, + canShare: false, + canDelete: false, + isFolder: false, + provisioned: false, + }, + }; + }) + ) + ); + } +} + export function getDashboardSnapshotSrv(): DashboardSnapshotSrv { + if (config.featureToggles.kubernetesSnapshots) { + return new K8sAPI(); + } return legacyDashboardSnapshotSrv; } diff --git a/public/openapi3.json b/public/openapi3.json index 5b63c944acf..b62fcb9db2b 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -4058,8 +4058,12 @@ }, "CreateDashboardSnapshotCommand": { "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, "dashboard": { - "$ref": "#/components/schemas/Json" + "$ref": "#/components/schemas/Unstructured" }, "deleteKey": { "description": "Unique key used to delete the snapshot. It is different from the `key` so that only the creator can delete the snapshot. Required if `external` is `true`.", @@ -4080,6 +4084,10 @@ "description": "Define the unique key. Required if `external` is `true`.", "type": "string" }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + }, "name": { "description": "Snapshot name", "type": "string" @@ -4422,6 +4430,41 @@ }, "type": "object" }, + "DashboardCreateCommand": { + "description": "These are the values expected to be sent from an end user\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, + "dashboard": { + "$ref": "#/components/schemas/Unstructured" + }, + "expires": { + "default": 0, + "description": "When the snapshot should expire in seconds in seconds. Default is never to expire.", + "format": "int64", + "type": "integer" + }, + "external": { + "default": false, + "description": "these are passed when storing an external snapshot ref\nSave the snapshot on an external server rather than locally.", + "type": "boolean" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + }, + "name": { + "description": "Snapshot name", + "type": "string" + } + }, + "required": [ + "dashboard" + ], + "type": "object" + }, "DashboardFullWithMeta": { "properties": { "dashboard": { @@ -11247,6 +11290,21 @@ "Type": { "type": "string" }, + "TypeMeta": { + "description": "+k8s:deepcopy-gen=false", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n+optional", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n+optional", + "type": "string" + } + }, + "title": "TypeMeta describes an individual object in an API response or request\nwith strings representing the type of the object and its API schema version.\nStructures that are versioned or persisted should inline TypeMeta.", + "type": "object" + }, "URL": { "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use the EscapedPath method, which preserves\nthe original encoding of Path.\n\nThe RawPath field is an optional field which is only set when the default\nencoding of Path is different from the escaped path. See the EscapedPath method\nfor more details.\n\nURL's String method uses the EscapedPath method to obtain the path.", "properties": { @@ -11287,6 +11345,17 @@ "title": "A URL represents a parsed URL (technically, a URI reference).", "type": "object" }, + "Unstructured": { + "description": "Unstructured allows objects that do not have Golang structs registered to be manipulated\ngenerically.", + "properties": { + "Object": { + "additionalProperties": false, + "description": "Object is a JSON compatible map with string, float, int, bool, []interface{},\nor map[string]interface{} children.", + "type": "object" + } + }, + "type": "object" + }, "UpdateAlertNotificationCommand": { "properties": { "disableResolveMessage": {