From 529987825380deba3094a6e5510f1fa51b16603e Mon Sep 17 00:00:00 2001 From: Victor Marin <36818606+mdvictor@users.noreply.github.com> Date: Fri, 19 Sep 2025 16:50:36 +0300 Subject: [PATCH 01/54] TimeRangePicker: Enhance panning keys (#111175) * extend timepicker to receive pan duration * i18n * scenes canary for testing * refactor * refactor * refactor * canary version * refactor * fix * bump scenes --- .../dashboard-cujs/dashboard-view.spec.ts | 4 ++-- package.json | 4 ++-- .../DateTimePickers/TimeRangePicker.tsx | 20 ++++++++++++----- yarn.lock | 22 +++++++++---------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/e2e-playwright/dashboard-cujs/dashboard-view.spec.ts b/e2e-playwright/dashboard-cujs/dashboard-view.spec.ts index 767ec2424e6..9fccafada83 100644 --- a/e2e-playwright/dashboard-cujs/dashboard-view.spec.ts +++ b/e2e-playwright/dashboard-cujs/dashboard-view.spec.ts @@ -134,8 +134,8 @@ test.describe( expect.soft(await timePickerButton.textContent()).toContain('2024-01-01 08:30:00 to 2024-01-01 08:40:00'); - const forwardBtn = page.locator('button[aria-label="Move time range forwards"]'); - const backwardBtn = page.locator('button[aria-label="Move time range backwards"]'); + const forwardBtn = page.locator('button[aria-label*="Move"][aria-label*="forward"]'); + const backwardBtn = page.locator('button[aria-label*="Move"][aria-label*="backward"]'); await forwardBtn.click(); diff --git a/package.json b/package.json index b5a74906856..3bece98c2ab 100644 --- a/package.json +++ b/package.json @@ -286,8 +286,8 @@ "@grafana/plugin-ui": "^0.10.10", "@grafana/prometheus": "workspace:*", "@grafana/runtime": "workspace:*", - "@grafana/scenes": "6.35.3", - "@grafana/scenes-react": "6.35.3", + "@grafana/scenes": "6.38.0", + "@grafana/scenes-react": "6.38.0", "@grafana/schema": "workspace:*", "@grafana/sql": "workspace:*", "@grafana/ui": "workspace:*", diff --git a/packages/grafana-ui/src/components/DateTimePickers/TimeRangePicker.tsx b/packages/grafana-ui/src/components/DateTimePickers/TimeRangePicker.tsx index 926fcce5615..3157b6ca837 100644 --- a/packages/grafana-ui/src/components/DateTimePickers/TimeRangePicker.tsx +++ b/packages/grafana-ui/src/components/DateTimePickers/TimeRangePicker.tsx @@ -55,6 +55,8 @@ export interface TimeRangePickerProps { onChangeFiscalYearStartMonth?: (month: number) => void; onMoveBackward: () => void; onMoveForward: () => void; + moveForwardTooltip?: string; + moveBackwardTooltip?: string; onZoom: () => void; onError?: (error?: string) => void; history?: TimeRange[]; @@ -78,6 +80,8 @@ export function TimeRangePicker(props: TimeRangePickerProps) { value, onMoveBackward, onMoveForward, + moveForwardTooltip, + moveBackwardTooltip, onZoom, onError, timeZone, @@ -150,11 +154,14 @@ export function TimeRangePicker(props: TimeRangePickerProps) { return ( @@ -207,12 +214,15 @@ export function TimeRangePicker(props: TimeRangePickerProps) { {timeSyncButton} diff --git a/yarn.lock b/yarn.lock index 914011568c1..4df524d5425 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3521,11 +3521,11 @@ __metadata: languageName: unknown linkType: soft -"@grafana/scenes-react@npm:6.35.3": - version: 6.35.3 - resolution: "@grafana/scenes-react@npm:6.35.3" +"@grafana/scenes-react@npm:6.38.0": + version: 6.38.0 + resolution: "@grafana/scenes-react@npm:6.38.0" dependencies: - "@grafana/scenes": "npm:6.35.3" + "@grafana/scenes": "npm:6.38.0" lru-cache: "npm:^10.2.2" react-use: "npm:^17.4.0" peerDependencies: @@ -3537,13 +3537,13 @@ __metadata: react: ^18.0.0 react-dom: ^18.0.0 react-router-dom: ^6.28.0 - checksum: 10/2c220eef6aa8560d073a27a02918b4bbf4d0cffee273ff597e94eb4376e75d0e47fcdaa2cd1ca8ea851c29477722dec2e2d2f817f2c5c185151de2051c5e2ab7 + checksum: 10/7b38553f1950f6592eb24878061cc7c45fc2aa1776c4b1cd521c6995d11f8beb0b2b275afda0ebf0ec19b7f7c79b77126caad4843bc9a75bd09881645e882c91 languageName: node linkType: hard -"@grafana/scenes@npm:6.35.3": - version: 6.35.3 - resolution: "@grafana/scenes@npm:6.35.3" +"@grafana/scenes@npm:6.38.0": + version: 6.38.0 + resolution: "@grafana/scenes@npm:6.38.0" dependencies: "@floating-ui/react": "npm:^0.26.16" "@leeoniya/ufuzzy": "npm:^1.0.16" @@ -3563,7 +3563,7 @@ __metadata: react: ^18.0.0 react-dom: ^18.0.0 react-router-dom: ^6.28.0 - checksum: 10/a209e6a1cbe6c4e70c2cd8c413df2560a59b157944888e317385f19e43e23ff826d84c922618b1268a7085278aca45a369c8f6255bf70a06134622c6ef2a542e + checksum: 10/9372896e76271becb80e4a8691d8c51bc3b54b54d98c82f2953134f407a7370dc66a3e8afaa508003147e2f1b5b08d0ddbb6b1616a3a3ab3f7ad69fb941dd998 languageName: node linkType: hard @@ -18199,8 +18199,8 @@ __metadata: "@grafana/plugin-ui": "npm:^0.10.10" "@grafana/prometheus": "workspace:*" "@grafana/runtime": "workspace:*" - "@grafana/scenes": "npm:6.35.3" - "@grafana/scenes-react": "npm:6.35.3" + "@grafana/scenes": "npm:6.38.0" + "@grafana/scenes-react": "npm:6.38.0" "@grafana/schema": "workspace:*" "@grafana/sql": "workspace:*" "@grafana/test-utils": "workspace:*" From 460f776e6fc84af59150f5dd936be32f4ed7a23b Mon Sep 17 00:00:00 2001 From: Alex Khomenko Date: Fri, 19 Sep 2025 16:50:47 +0300 Subject: [PATCH 02/54] Provisioning: Skip folder sync when repository is empty (#111371) * Provisioning: Skip folder sync when repository is empty * Simplify condition * Fix condition --- .../app/features/provisioning/Wizard/ProvisioningWizard.tsx | 4 ++-- .../features/provisioning/Wizard/hooks/useResourceStats.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/public/app/features/provisioning/Wizard/ProvisioningWizard.tsx b/public/app/features/provisioning/Wizard/ProvisioningWizard.tsx index 717400785fc..0a7d0cf7ca6 100644 --- a/public/app/features/provisioning/Wizard/ProvisioningWizard.tsx +++ b/public/app/features/provisioning/Wizard/ProvisioningWizard.tsx @@ -101,14 +101,14 @@ export function ProvisioningWizard({ type }: { type: RepoType }) { handleSubmit, } = methods; - const [repoName = '', repoType] = watch(['repositoryName', 'repository.type']); + const [repoName = '', repoType, syncTarget] = watch(['repositoryName', 'repository.type', 'repository.sync.target']); const [submitData] = useCreateOrUpdateRepository(repoName); const [deleteRepository] = useDeleteRepositoryMutation(); const { shouldSkipSync, requiresMigration, isLoading: isResourceStatsLoading, - } = useResourceStats(repoName, isLegacyStorage); + } = useResourceStats(repoName, isLegacyStorage, syncTarget); const { createSyncJob, isLoading: isCreatingSkipJob } = useCreateSyncJob({ repoName: repoName, requiresMigration, diff --git a/public/app/features/provisioning/Wizard/hooks/useResourceStats.ts b/public/app/features/provisioning/Wizard/hooks/useResourceStats.ts index edf90010ce4..4afc252e0e5 100644 --- a/public/app/features/provisioning/Wizard/hooks/useResourceStats.ts +++ b/public/app/features/provisioning/Wizard/hooks/useResourceStats.ts @@ -6,6 +6,7 @@ import { GetRepositoryFilesApiResponse, GetResourceStatsApiResponse, ManagerStats, + RepositoryView, ResourceCount, useGetRepositoryFilesQuery, useGetResourceStatsQuery, @@ -76,7 +77,7 @@ function getResourceStats(files?: GetRepositoryFilesApiResponse, stats?: GetReso /** * Hook that provides resource statistics and sync logic */ -export function useResourceStats(repoName?: string, isLegacyStorage?: boolean) { +export function useResourceStats(repoName?: string, isLegacyStorage?: boolean, syncTarget?: RepositoryView['target']) { const resourceStatsQuery = useGetResourceStatsQuery(repoName ? undefined : skipToken); const filesQuery = useGetRepositoryFilesQuery(repoName ? { name: repoName } : skipToken); @@ -96,7 +97,7 @@ export function useResourceStats(repoName?: string, isLegacyStorage?: boolean) { }, [resourceStatsQuery.data]); const requiresMigration = isLegacyStorage || resourceCount > 0; - const shouldSkipSync = !requiresMigration && resourceCount === 0 && fileCount === 0; + const shouldSkipSync = !isLegacyStorage && (resourceCount === 0 || syncTarget === 'folder') && fileCount === 0; // Format display strings const resourceCountDisplay = From 00c91459298a82853c4a99a855883d73987d0dc2 Mon Sep 17 00:00:00 2001 From: Tobias Skarhed <1438972+tskarhed@users.noreply.github.com> Date: Fri, 19 Sep 2025 15:51:24 +0200 Subject: [PATCH 03/54] Scopes: Selector search highlighting (#111295) * Add scopes search highlight with wildcard support * Change so query is not being applied to parent * Update e2e tests with more stable selectors * Fix initial load query * Use the Highlighter library instead * Remove unused sanitization * Remove undefined export --- .../dashboard-cujs/scope-cujs.spec.ts | 7 +- e2e-playwright/utils/scope-helpers.ts | 10 +- .../selector/ScopesSelectorService.test.ts | 117 ++++++++++++++++++ .../scopes/selector/ScopesSelectorService.ts | 46 ++++--- .../scopes/selector/ScopesTreeItem.tsx | 74 +++++++++-- 5 files changed, 219 insertions(+), 35 deletions(-) diff --git a/e2e-playwright/dashboard-cujs/scope-cujs.spec.ts b/e2e-playwright/dashboard-cujs/scope-cujs.spec.ts index 25f4a9233d5..919930027b8 100644 --- a/e2e-playwright/dashboard-cujs/scope-cujs.spec.ts +++ b/e2e-playwright/dashboard-cujs/scope-cujs.spec.ts @@ -169,12 +169,15 @@ test.describe( await searchScopes(page, scopeSearchOne, [secondLevelScopes[0]]); await expect.soft(scopeTreeCheckboxes).toHaveCount(1); - expect.soft(await scopeTreeCheckboxes.first().locator('../..').textContent()).toBe(scopeSearchOne); + + // Check grandparent title, to make sure we get what we want + expect.soft(await scopeTreeCheckboxes.first().locator('../../..').textContent()).toBe(scopeSearchOne); await searchScopes(page, scopeSearchTwo, [secondLevelScopes[1]]); await expect.soft(scopeTreeCheckboxes).toHaveCount(1); - expect.soft(await scopeTreeCheckboxes.first().locator('../..').textContent()).toBe(scopeSearchTwo); + // Check grandparent title, to make sure we get what we want + expect.soft(await scopeTreeCheckboxes.first().locator('../../..').textContent()).toBe(scopeSearchTwo); }); }); } diff --git a/e2e-playwright/utils/scope-helpers.ts b/e2e-playwright/utils/scope-helpers.ts index 2c27697f85d..300350e4c47 100644 --- a/e2e-playwright/utils/scope-helpers.ts +++ b/e2e-playwright/utils/scope-helpers.ts @@ -262,13 +262,15 @@ export async function getScopeLeafName(page: Page, nth: number): Promise } export async function getScopeLeafTitle(page: Page, nth: number): Promise { - const locator = page.getByTestId(/^scopes-tree-.*-(checkbox|radio)/).nth(nth); - const scopeTitle = await locator.locator('../..').textContent(); - + const leafLocator = page.getByTestId(/^scopes-tree-.*-(checkbox|radio)/).nth(nth); + // Find the closest ancestor element that has the main tree item test id + const titleLocator = leafLocator.locator( + 'xpath=ancestor::*[@data-testid][starts-with(@data-testid, "scopes-tree-") and not(contains(@data-testid, "-checkbox")) and not(contains(@data-testid, "-radio")) and not(contains(@data-testid, "-expand"))]' + ); + const scopeTitle = await titleLocator.textContent(); if (!scopeTitle) { throw new Error('There are no scopes in the selector'); } - return scopeTitle; } diff --git a/public/app/features/scopes/selector/ScopesSelectorService.test.ts b/public/app/features/scopes/selector/ScopesSelectorService.test.ts index f1e7265e735..c9ef8afadbc 100644 --- a/public/app/features/scopes/selector/ScopesSelectorService.test.ts +++ b/public/app/features/scopes/selector/ScopesSelectorService.test.ts @@ -98,6 +98,7 @@ describe('ScopesSelectorService', () => { }); it('should update node query and fetch children when query changes', async () => { + await service.updateNode('', true, ''); // Expand first await service.updateNode('', true, 'new-query'); expect(service.state.tree).toMatchObject({ children: {}, @@ -116,6 +117,122 @@ describe('ScopesSelectorService', () => { // Only the first expansion should trigger fetchNodes expect(apiClient.fetchNodes).toHaveBeenCalledTimes(1); }); + + it('should clear query on first expansion but keep it when filtering within populated node', async () => { + const mockChildNode: ScopeNode = { + metadata: { name: 'child-node' }, + spec: { linkId: 'child-scope', linkType: 'scope', parentName: '', nodeType: 'leaf', title: 'child-node' }, + }; + + apiClient.fetchNodes.mockResolvedValue([mockChildNode]); + + // Scenario 1: First expansion (no children yet) - clear query for unfiltered view + await service.updateNode('', true, 'search-query'); + expect(apiClient.fetchNodes).toHaveBeenCalledWith({ parent: '', query: undefined }); + + // Parent query should be cleared and child nodes should have no query (first expansion) + expect(service.state.tree?.query).toBe(''); + let childTreeNode = service.state.tree?.children?.['child-node']; + expect(childTreeNode?.query).toBe(''); + + // Scenario 2: Filtering within node that already has children + await service.updateNode('', true, 'new-search'); + expect(apiClient.fetchNodes).toHaveBeenCalledWith({ parent: '', query: 'new-search' }); + + // Parent and child nodes should have the filter query (filtering within existing children) + expect(service.state.tree?.query).toBe('new-search'); + childTreeNode = service.state.tree?.children?.['child-node']; + expect(childTreeNode?.query).toBe('new-search'); + + expect(apiClient.fetchNodes).toHaveBeenCalledTimes(2); + }); + + it('should always reset query on any expansion', async () => { + const mockChildNode: ScopeNode = { + metadata: { name: 'child-node' }, + spec: { linkId: 'child-scope', linkType: 'scope', parentName: '', nodeType: 'leaf', title: 'child-node' }, + }; + + apiClient.fetchNodes.mockResolvedValue([mockChildNode]); + + // First expansion with any query should reset parent query and not pass query to API + await service.updateNode('', true, 'some-search-query'); + + // Verify query is reset and API called without query for first expansion + expect(service.state.tree?.query).toBe(''); + expect(apiClient.fetchNodes).toHaveBeenCalledWith({ parent: '', query: undefined }); + expect(service.state.tree?.children?.['child-node']?.query).toBe(''); + }); + + it('should handle query reset correctly for nested levels beyond root', async () => { + // Set up mock nodes for multi-level hierarchy + const mockParentNode: ScopeNode = { + metadata: { name: 'parent-container' }, + spec: { linkId: '', linkType: 'scope', parentName: '', nodeType: 'container', title: 'Parent Container' }, + }; + + const mockChildNode: ScopeNode = { + metadata: { name: 'child-container' }, + spec: { + linkId: '', + linkType: 'scope', + parentName: 'parent-container', + nodeType: 'container', + title: 'Child Container', + }, + }; + + const mockGrandchildNode: ScopeNode = { + metadata: { name: 'grandchild-leaf' }, + spec: { + linkId: 'leaf-scope', + linkType: 'scope', + parentName: 'child-container', + nodeType: 'leaf', + title: 'Grandchild Leaf', + }, + }; + + // Mock different responses for different parent nodes + apiClient.fetchNodes.mockImplementation((options: { parent?: string; query?: string; limit?: number }) => { + if (options.parent === '') { + return Promise.resolve([mockParentNode]); + } else if (options.parent === 'parent-container') { + return Promise.resolve([mockChildNode]); + } else if (options.parent === 'child-container') { + return Promise.resolve([mockGrandchildNode]); + } + return Promise.resolve([]); + }); + + // Step 1: Expand root node with search query + await service.updateNode('', true, 'search-query'); + + // Root should have query reset, API called without query + expect(service.state.tree?.query).toBe(''); + expect(apiClient.fetchNodes).toHaveBeenCalledWith({ parent: '', query: undefined }); + expect(service.state.tree?.children?.['parent-container']?.query).toBe(''); + + // Step 2: Expand first-level child with search query + await service.updateNode('parent-container', true, 'open-search-query'); + + // First-level child should have query reset, API called without query + const parentContainer = service.state.tree?.children?.['parent-container']; + expect(parentContainer?.query).toBe(''); + expect(apiClient.fetchNodes).toHaveBeenCalledWith({ parent: 'parent-container', query: undefined }); + expect(parentContainer?.children?.['child-container']?.query).toBe(''); + + // Step 3: Now filter within the first-level child (second call to same node) + await service.updateNode('parent-container', true, 'filter-search'); + + // Now both parent and children should show the filter query since we're filtering within existing children + const newParentContainer = service.state.tree?.children?.['parent-container']; + expect(newParentContainer?.query).toBe('filter-search'); + expect(apiClient.fetchNodes).toHaveBeenCalledWith({ parent: 'parent-container', query: 'filter-search' }); + expect(newParentContainer?.children?.['child-container']?.query).toBe('filter-search'); + + expect(apiClient.fetchNodes).toHaveBeenCalledTimes(3); + }); }); describe('selectScope and deselectScope', () => { diff --git a/public/app/features/scopes/selector/ScopesSelectorService.ts b/public/app/features/scopes/selector/ScopesSelectorService.ts index 6c6493ca16d..5d8bf456f6a 100644 --- a/public/app/features/scopes/selector/ScopesSelectorService.ts +++ b/public/app/features/scopes/selector/ScopesSelectorService.ts @@ -98,21 +98,28 @@ export class ScopesSelectorService extends ScopesServiceBase { - treeNode.expanded = true; - treeNode.query = query || ''; - }); - this.updateState({ tree: newTree }); - await this.loadNodeChildren(path, nodeToExpand, query); - } - } else { - throw new Error(`Trying to expand node at id ${scopeNodeId} that is not expandable`); - } - } else { - throw new Error(`Trying to expand node at id ${scopeNodeId} not found`); + if (!nodeToExpand) { + throw new Error(`Node ${scopeNodeId} not found in tree`); + } + + if (nodeToExpand.scopeNodeId !== '' && !isNodeExpandable(this.state.nodes[nodeToExpand.scopeNodeId])) { + throw new Error(`Trying to expand node at id ${scopeNodeId} that is not expandable`); + } + + // Check if this is first expansion or filtering within existing children + const haveChildrenLoaded = nodeToExpand.children && Object.keys(nodeToExpand.children).length > 0; + + if (!nodeToExpand.expanded || nodeToExpand.query !== query || !haveChildrenLoaded) { + const newTree = modifyTreeNodeAtPath(this.state.tree!, path, (treeNode) => { + treeNode.expanded = true; + // Reset query on first expansion, keep it only when filtering within existing children + treeNode.query = ''; + }); + this.updateState({ tree: newTree }); + + // For API call: only pass query if filtering within existing children + const queryForAPI = haveChildrenLoaded ? query : query === '' ? '' : undefined; + await this.loadNodeChildren(path, nodeToExpand, queryForAPI, haveChildrenLoaded); } }; @@ -131,10 +138,9 @@ export class ScopesSelectorService extends ScopesServiceBase { + private loadNodeChildren = async (path: string[], treeNode: TreeNode, query?: string, haveChildrenLoaded = false) => { this.updateState({ loadingNodeName: treeNode.scopeNodeId }); - // We are expanding node that wasn't yet expanded so we don't have any query to filter by yet. const childNodes = await this.apiClient.fetchNodes({ parent: treeNode.scopeNodeId, query }); const newNodes = { ...this.state.nodes }; @@ -144,12 +150,15 @@ export class ScopesSelectorService extends ScopesServiceBase { + // Set parent query only when filtering within existing children + treeNode.query = haveChildrenLoaded ? query || '' : ''; treeNode.children = {}; for (const node of childNodes) { treeNode.children[node.metadata.name] = { expanded: false, scopeNodeId: node.metadata.name, - query: '', + // Only set query on tree nodes if parent already has children (filtering vs first expansion). This is used for saerch highlighting. + query: haveChildrenLoaded ? query || '' : '', children: undefined, }; } @@ -233,6 +242,7 @@ export class ScopesSelectorService extends ScopesServiceBase { if (expanded) { return this.expandOrFilterNode(scopeNodeId, query); diff --git a/public/app/features/scopes/selector/ScopesTreeItem.tsx b/public/app/features/scopes/selector/ScopesTreeItem.tsx index 12e1d8dc3f2..2fb2fc0026f 100644 --- a/public/app/features/scopes/selector/ScopesTreeItem.tsx +++ b/public/app/features/scopes/selector/ScopesTreeItem.tsx @@ -1,4 +1,5 @@ import { css, cx } from '@emotion/css'; +import Highlighter from 'react-highlight-words'; import { GrafanaTheme2 } from '@grafana/data'; import { t } from '@grafana/i18n'; @@ -51,6 +52,12 @@ export function ScopesTreeItem({ const isSelectable = isNodeSelectable(scopeNode); const isExpandable = isNodeExpandable(scopeNode); + // Create search words for highlighting if there's a query + // Only highlight if we have a query AND this node is not expanded (not a parent showing children) + const titleText = scopeNode.spec.title; + const shouldHighlight = treeNode.query && !treeNode.expanded; + const searchWords = shouldHighlight ? getSearchWordsFromQuery(treeNode.query) : []; + return (
{isSelectable && !treeNode.expanded ? ( disableMultiSelect ? ( @@ -74,22 +82,41 @@ export function ScopesTreeItem({ id={treeNode.scopeNodeId} name={treeNode.scopeNodeId} checked={selected} - label={isExpandable ? '' : scopeNode.spec.title} + label={ + isExpandable ? ( + '' + ) : shouldHighlight ? ( + + ) : ( + titleText + ) + } data-testid={`scopes-tree-${treeNode.scopeNodeId}-radio`} onClick={() => { selected ? deselectScope(treeNode.scopeNodeId) : selectScope(treeNode.scopeNodeId); }} /> ) : ( - { - selected ? deselectScope(treeNode.scopeNodeId) : selectScope(treeNode.scopeNodeId); - }} - /> +
+ { + selected ? deselectScope(treeNode.scopeNodeId) : selectScope(treeNode.scopeNodeId); + }} + /> + {!isExpandable && ( + + )} +
) ) : null} @@ -104,7 +131,11 @@ export function ScopesTreeItem({ > - {scopeNode.spec.title} + {shouldHighlight ? ( + + ) : ( + titleText + )} )}
@@ -126,6 +157,15 @@ export function ScopesTreeItem({ ); } +// Convert a query string with wildcards into search words for react-highlight-words +function getSearchWordsFromQuery(query: string): string[] { + if (!query) { + return []; + } + // Split query string on wildcard and filter out empty parts + return query.split('*').filter((part) => part.length > 0); +} + export const getTreeItemElementId = (scopeNodeId?: string) => { return scopeNodeId ? `scopes-tree-item-${scopeNodeId}` : undefined; }; @@ -159,6 +199,18 @@ const getStyles = (theme: GrafanaTheme2) => { // Fix for checkboxes and radios outline overflow due to scrollbars paddingLeft: theme.spacing(0.5), }), + checkboxWithLabel: css({ + alignItems: 'center', + display: 'flex', + gap: theme.spacing(1), + }), + checkboxLabel: css({ + fontSize: theme.typography.pxToRem(14), + lineHeight: theme.typography.pxToRem(22), + fontWeight: theme.typography.fontWeightRegular, + cursor: 'pointer', + margin: 0, + }), expand: css({ alignItems: 'center', background: 'none', From 047b40fb544597ae6a813c6728d835c034be56f7 Mon Sep 17 00:00:00 2001 From: Andreas Christou Date: Fri, 19 Sep 2025 16:08:49 +0200 Subject: [PATCH 04/54] Azure: Increase test timeout (#111370) Increase test timeout --- .../azuremonitor/components/TracesQueryEditor/Filters.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/datasource/azuremonitor/components/TracesQueryEditor/Filters.test.tsx b/public/app/plugins/datasource/azuremonitor/components/TracesQueryEditor/Filters.test.tsx index b6154d9205e..89d109c4600 100644 --- a/public/app/plugins/datasource/azuremonitor/components/TracesQueryEditor/Filters.test.tsx +++ b/public/app/plugins/datasource/azuremonitor/components/TracesQueryEditor/Filters.test.tsx @@ -27,6 +27,7 @@ jest.mock('@grafana/runtime', () => { }), }; }); +jest.setTimeout(10000); const variableOptionGroup = { label: 'Template variables', From c20fdbe39e6c83c24b2b06d0bf896dd82aece674 Mon Sep 17 00:00:00 2001 From: Misi Date: Fri, 19 Sep 2025 16:09:28 +0200 Subject: [PATCH 05/54] IAM: Fix display_query.sql (#111388) Fix --- pkg/registry/apis/iam/legacy/display_query.sql | 2 +- .../iam/legacy/testdata/mysql--display_query-display_ids.sql | 2 +- .../legacy/testdata/mysql--display_query-display_ids_uids.sql | 2 +- .../iam/legacy/testdata/mysql--display_query-display_uids.sql | 2 +- .../iam/legacy/testdata/postgres--display_query-display_ids.sql | 2 +- .../testdata/postgres--display_query-display_ids_uids.sql | 2 +- .../legacy/testdata/postgres--display_query-display_uids.sql | 2 +- .../iam/legacy/testdata/sqlite--display_query-display_ids.sql | 2 +- .../legacy/testdata/sqlite--display_query-display_ids_uids.sql | 2 +- .../iam/legacy/testdata/sqlite--display_query-display_uids.sql | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/registry/apis/iam/legacy/display_query.sql b/pkg/registry/apis/iam/legacy/display_query.sql index 15dfff35d09..3e8c1e33112 100644 --- a/pkg/registry/apis/iam/legacy/display_query.sql +++ b/pkg/registry/apis/iam/legacy/display_query.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM {{ .Ident .UserTable }} as u JOIN {{ .Ident .OrgUserTable }} as o ON u.id = o.user_id WHERE o.org_id = {{ .Arg .Query.OrgID }} AND ( 1=2 {{ if .Query.UIDs }} diff --git a/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids.sql b/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids.sql index 66f977fe3db..7354603168a 100755 --- a/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM `grafana`.`user` as u JOIN `grafana`.`org_user` as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR u.id IN (1, 2) diff --git a/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids_uids.sql b/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids_uids.sql index 933d45adbee..ec1927fe7c6 100755 --- a/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids_uids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_ids_uids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM `grafana`.`user` as u JOIN `grafana`.`org_user` as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR uid IN ('a', 'b') diff --git a/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_uids.sql b/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_uids.sql index a1c7c11dcad..6353dd6f6b9 100755 --- a/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_uids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/mysql--display_query-display_uids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM `grafana`.`user` as u JOIN `grafana`.`org_user` as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR uid IN ('a', 'b') diff --git a/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids.sql b/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids.sql index d3cb1fb1e8c..0d2ccd8befa 100755 --- a/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM "grafana"."user" as u JOIN "grafana"."org_user" as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR u.id IN (1, 2) diff --git a/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids_uids.sql b/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids_uids.sql index dfb4ed5afe2..a1514263169 100755 --- a/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids_uids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_ids_uids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM "grafana"."user" as u JOIN "grafana"."org_user" as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR uid IN ('a', 'b') diff --git a/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_uids.sql b/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_uids.sql index a6989bb94e0..66a204fc20d 100755 --- a/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_uids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/postgres--display_query-display_uids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM "grafana"."user" as u JOIN "grafana"."org_user" as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR uid IN ('a', 'b') diff --git a/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids.sql b/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids.sql index d3cb1fb1e8c..0d2ccd8befa 100755 --- a/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM "grafana"."user" as u JOIN "grafana"."org_user" as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR u.id IN (1, 2) diff --git a/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids_uids.sql b/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids_uids.sql index dfb4ed5afe2..a1514263169 100755 --- a/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids_uids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_ids_uids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM "grafana"."user" as u JOIN "grafana"."org_user" as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR uid IN ('a', 'b') diff --git a/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_uids.sql b/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_uids.sql index a6989bb94e0..66a204fc20d 100755 --- a/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_uids.sql +++ b/pkg/registry/apis/iam/legacy/testdata/sqlite--display_query-display_uids.sql @@ -1,6 +1,6 @@ SELECT o.org_id, u.id, u.uid, u.login, u.email, u.name, u.created, u.updated, u.is_service_account, u.is_disabled, u.is_admin, - u.email_verified, u.is_provisioned, u.last_seen_at + u.email_verified, u.is_provisioned, u.last_seen_at, o.role FROM "grafana"."user" as u JOIN "grafana"."org_user" as o ON u.id = o.user_id WHERE o.org_id = 2 AND ( 1=2 OR uid IN ('a', 'b') From 3dc30b5acbddfa61f17a0e131b3b308abe3a995a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 19 Sep 2025 16:10:33 +0200 Subject: [PATCH 06/54] Buttons: Active style for buttons (#111235) * chore: initial tweaks * chore: remove shadow * chore: add reduced motion support, clean up * chore: clean * chore: active states tweaks * Buttons: Add active state / style * Update ToolbarButton and IconButton * Update * Update * Get rid of important on disabled styles * Only color for active * Remove transform bits --------- Co-authored-by: Galen --- .../src/components/Button/Button.tsx | 57 ++++++++++++------- .../src/components/IconButton/IconButton.tsx | 33 +++++++---- .../ToolbarButton/ToolbarButton.tsx | 16 ++++-- packages/grafana-ui/src/themes/mixins.ts | 7 +++ 4 files changed, 78 insertions(+), 35 deletions(-) diff --git a/packages/grafana-ui/src/components/Button/Button.tsx b/packages/grafana-ui/src/components/Button/Button.tsx index 83b1c5825c0..c596904a39b 100644 --- a/packages/grafana-ui/src/components/Button/Button.tsx +++ b/packages/grafana-ui/src/components/Button/Button.tsx @@ -5,7 +5,7 @@ import * as React from 'react'; import { GrafanaTheme2, ThemeRichColor } from '@grafana/data'; import { useTheme2 } from '../../themes/ThemeContext'; -import { getFocusStyles, getMouseFocusStyles } from '../../themes/mixins'; +import { getButtonFocusStyles, getMouseFocusStyles } from '../../themes/mixins'; import { IconName, IconSize, IconType } from '../../types/icon'; import { ComponentSize } from '../../types/size'; import { getPropertiesForButtonSize } from '../Forms/commonStyles'; @@ -234,7 +234,7 @@ export const getButtonStyles = (props: StyleProps) => { const { height, padding, fontSize } = getPropertiesForButtonSize(size, theme); const variantStyles = getPropertiesForVariant(theme, variant, fill); const disabledStyles = getPropertiesForDisabled(theme, variant, fill); - const focusStyle = getFocusStyles(theme); + const focusStyle = getButtonFocusStyles(theme); const paddingMinusBorder = theme.spacing.gridSize * padding - 1; return { @@ -263,6 +263,12 @@ export const getButtonStyles = (props: StyleProps) => { ...variantStyles, ':disabled': disabledStyles, '&[disabled]': disabledStyles, + + [theme.transitions.handleMotion('no-preference', 'reduce')]: { + transition: theme.transitions.create(['background-color', 'border-color', 'color'], { + duration: theme.transitions.duration.short, + }), + }, }), disabled: css(disabledStyles, { '&:hover': css(disabledStyles), @@ -290,12 +296,18 @@ export const getButtonStyles = (props: StyleProps) => { }; }; -function getButtonVariantStyles(theme: GrafanaTheme2, color: ThemeRichColor, fill: ButtonFill) { +export function getActiveButtonStyles(color: ThemeRichColor, fill: ButtonFill) { + return { + background: fill === 'solid' ? color.main : 'transparent', + }; +} + +export function getButtonVariantStyles(theme: GrafanaTheme2, color: ThemeRichColor, fill: ButtonFill) { let outlineBorderColor = color.border; let borderColor = 'transparent'; let hoverBorderColor = 'transparent'; - // Secondary button has some special rules as we lack theem color token to + // Secondary button has some special rules as we lack the color token to // specify border color for normal button vs border color for outline button if (color.name === 'secondary') { borderColor = color.border; @@ -308,15 +320,16 @@ function getButtonVariantStyles(theme: GrafanaTheme2, color: ThemeRichColor, fil background: 'transparent', color: color.text, border: `1px solid ${outlineBorderColor}`, - transition: theme.transitions.create(['background-color', 'border-color', 'color'], { - duration: theme.transitions.duration.short, - }), - '&:hover': { + '&:hover, &:focus': { background: color.transparent, borderColor: theme.colors.emphasize(outlineBorderColor, 0.25), color: color.text, }, + + '&:active': { + ...getActiveButtonStyles(color, fill), + }, }; } @@ -325,18 +338,15 @@ function getButtonVariantStyles(theme: GrafanaTheme2, color: ThemeRichColor, fil background: 'transparent', color: color.text, border: '1px solid transparent', - transition: theme.transitions.create(['background-color', 'color'], { - duration: theme.transitions.duration.short, - }), - '&:focus': { - outline: 'none', - textDecoration: 'none', - }, - - '&:hover': { + '&:hover, &:focus': { background: color.transparent, textDecoration: 'none', + outline: 'none', + }, + + '&:active': { + ...getActiveButtonStyles(color, fill), }, }; } @@ -345,9 +355,6 @@ function getButtonVariantStyles(theme: GrafanaTheme2, color: ThemeRichColor, fil background: color.main, color: color.contrastText, border: `1px solid ${borderColor}`, - transition: theme.transitions.create(['background-color', 'box-shadow', 'border-color', 'color'], { - duration: theme.transitions.duration.short, - }), '&:hover': { background: color.shade, @@ -355,6 +362,15 @@ function getButtonVariantStyles(theme: GrafanaTheme2, color: ThemeRichColor, fil boxShadow: theme.shadows.z1, borderColor: hoverBorderColor, }, + + '&:focus': { + background: color.shade, + color: color.contrastText, + }, + + '&:active': { + ...getActiveButtonStyles(color, fill), + }, }; } @@ -364,6 +380,7 @@ function getPropertiesForDisabled(theme: GrafanaTheme2, variant: ButtonVariant, boxShadow: 'none', color: theme.colors.text.disabled, transition: 'none', + background: theme.colors.action.disabledBackground, }; if (fill === 'text') { diff --git a/packages/grafana-ui/src/components/IconButton/IconButton.tsx b/packages/grafana-ui/src/components/IconButton/IconButton.tsx index 2964143b331..bbc4519f71f 100644 --- a/packages/grafana-ui/src/components/IconButton/IconButton.tsx +++ b/packages/grafana-ui/src/components/IconButton/IconButton.tsx @@ -1,13 +1,13 @@ import { css, cx } from '@emotion/css'; import * as React from 'react'; -import { GrafanaTheme2, colorManipulator, deprecationWarning } from '@grafana/data'; +import { GrafanaTheme2, deprecationWarning } from '@grafana/data'; import { useStyles2 } from '../../themes/ThemeContext'; import { getFocusStyles, getMouseFocusStyles } from '../../themes/mixins'; import { IconName, IconSize, IconType } from '../../types/icon'; import { ComponentSize } from '../../types/size'; -import { IconRenderer } from '../Button/Button'; +import { getActiveButtonStyles, IconRenderer } from '../Button/Button'; import { getSvgSize } from '../Icon/utils'; import { Tooltip } from '../Tooltip/Tooltip'; import { PopoverContent, TooltipPlacement } from '../Tooltip/types'; @@ -107,13 +107,17 @@ const getStyles = (theme: GrafanaTheme2, size: IconSize, variant: IconButtonVari // overall size of the IconButton on hover // theme.spacing.gridSize originates from 2*4px for padding and letting the IconSize generally decide on the hoverSize const hoverSize = getSvgSize(size) + theme.spacing.gridSize; + const activeButtonStyle = getActiveButtonStyles(theme.colors.secondary, 'text'); - let iconColor = theme.colors.text.primary; + let iconColor = theme.colors.primary.text; + let hoverColor = theme.colors.primary.transparent; - if (variant === 'primary') { - iconColor = theme.colors.primary.text; + if (variant === 'secondary') { + iconColor = theme.colors.secondary.text; + hoverColor = theme.colors.secondary.transparent; } else if (variant === 'destructive') { iconColor = theme.colors.error.text; + hoverColor = theme.colors.error.transparent; } return { @@ -129,11 +133,21 @@ const getStyles = (theme: GrafanaTheme2, size: IconSize, variant: IconButtonVari alignItems: 'center', padding: 0, color: iconColor, + borderRadius: theme.shape.radius.default, + + '&:active': { + '&:before, &:hover:before': { + backgroundColor: activeButtonStyle.background, + }, + }, '&[disabled], &:disabled': { cursor: 'not-allowed', color: theme.colors.action.disabledText, opacity: 0.65, + '&:hover:before': { + backgroundColor: 'transparent', + }, }, '&:before': { @@ -155,12 +169,9 @@ const getStyles = (theme: GrafanaTheme2, size: IconSize, variant: IconButtonVari '&:focus:not(:focus-visible)': getMouseFocusStyles(theme), - '&:hover': { - '&:before': { - backgroundColor: - variant === 'secondary' ? theme.colors.action.hover : colorManipulator.alpha(iconColor, 0.12), - opacity: 1, - }, + '&:hover:before': { + backgroundColor: hoverColor, + opacity: 1, }, }), icon: css({ diff --git a/packages/grafana-ui/src/components/ToolbarButton/ToolbarButton.tsx b/packages/grafana-ui/src/components/ToolbarButton/ToolbarButton.tsx index a866ccefc9e..376b1b3fc38 100644 --- a/packages/grafana-ui/src/components/ToolbarButton/ToolbarButton.tsx +++ b/packages/grafana-ui/src/components/ToolbarButton/ToolbarButton.tsx @@ -8,7 +8,7 @@ import { selectors } from '@grafana/e2e-selectors'; import { useStyles2 } from '../../themes/ThemeContext'; import { getFocusStyles, getMouseFocusStyles, mediaUp } from '../../themes/mixins'; import { IconSize } from '../../types/icon'; -import { getPropertiesForVariant } from '../Button/Button'; +import { getActiveButtonStyles, getPropertiesForVariant } from '../Button/Button'; import { Icon } from '../Icon/Icon'; import { Tooltip } from '../Tooltip/Tooltip'; @@ -134,11 +134,15 @@ const getStyles = (theme: GrafanaTheme2) => { color: theme.colors.text.primary, background: theme.colors.secondary.main, - '&:hover': { + '&:hover, &:focus': { color: theme.colors.text.primary, background: theme.colors.secondary.shade, border: `1px solid ${theme.colors.border.medium}`, }, + + '&:active': { + ...getActiveButtonStyles(theme.colors.secondary, 'solid'), + }, }); return { @@ -155,7 +159,7 @@ const getStyles = (theme: GrafanaTheme2) => { border: `1px solid ${theme.colors.secondary.border}`, whiteSpace: 'nowrap', [theme.transitions.handleMotion('no-preference', 'reduce')]: { - transition: theme.transitions.create(['background', 'box-shadow', 'border-color', 'color'], { + transition: theme.transitions.create(['background-color', 'border-color', 'color'], { duration: theme.transitions.duration.short, }), }, @@ -189,10 +193,14 @@ const getStyles = (theme: GrafanaTheme2) => { background: 'transparent', border: `1px solid transparent`, - '&:hover': { + '&:hover, &:focus': { color: theme.colors.text.primary, background: theme.colors.action.hover, }, + + '&:active': { + ...getActiveButtonStyles(theme.colors.secondary, 'solid'), + }, }), canvas: defaultOld, active: cx( diff --git a/packages/grafana-ui/src/themes/mixins.ts b/packages/grafana-ui/src/themes/mixins.ts index 5127047ada0..39bb7b19c83 100644 --- a/packages/grafana-ui/src/themes/mixins.ts +++ b/packages/grafana-ui/src/themes/mixins.ts @@ -72,6 +72,13 @@ export function getFocusStyles(theme: GrafanaTheme2) { }; } +export function getButtonFocusStyles(theme: GrafanaTheme2) { + return { + ...getFocusStyles(theme), + transitionProperty: undefined, + }; +} + // max-width is set up based on .grafana-tooltip class that's used in dashboard export const getTooltipContainerStyles = (theme: GrafanaTheme2) => ({ overflow: 'hidden', From 0db140e69725a3cf1bd65045745b039a6c56a568 Mon Sep 17 00:00:00 2001 From: Victor Marin <36818606+mdvictor@users.noreply.github.com> Date: Fri, 19 Sep 2025 17:41:14 +0300 Subject: [PATCH 07/54] Dashboard: New measurements on variable timings (#110709) * create new interaction profiler behaviour and also use it to measure scopes fetching times * PR mods * refactor * add canary scenes * refactor * refactor * canary version * add config flag for scope measurements * refactor * fix --- .../transformSaveModelSchemaV2ToScene.ts | 10 ++++ .../transformSaveModelToScene.ts | 10 ++++ .../dashboard/services/DashboardProfiler.ts | 20 +++++++ .../scopes/selector/ScopesSelectorService.ts | 52 ++++++++++++------- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/public/app/features/dashboard-scene/serialization/transformSaveModelSchemaV2ToScene.ts b/public/app/features/dashboard-scene/serialization/transformSaveModelSchemaV2ToScene.ts index 2a4c05839c2..3162c7282dd 100644 --- a/public/app/features/dashboard-scene/serialization/transformSaveModelSchemaV2ToScene.ts +++ b/public/app/features/dashboard-scene/serialization/transformSaveModelSchemaV2ToScene.ts @@ -172,6 +172,15 @@ export function transformSaveModelSchemaV2ToScene(dto: DashboardWithAccessInfo uid === '*' || uid === metadata.name) !== -1, + onInteractionComplete: getDashboardInteractionCallback(metadata.name, dashboard.title), + }, + getDashboardSceneProfiler() + ); + const dashboardScene = new DashboardScene( { description: dashboard.description, @@ -200,6 +209,7 @@ export function transformSaveModelSchemaV2ToScene(dto: DashboardWithAccessInfo uid === '*' || uid === oldModel.uid) !== -1, + onInteractionComplete: getDashboardInteractionCallback(oldModel.uid, oldModel.title), + }, + getDashboardSceneProfiler() + ); + const behaviorList: SceneObjectState['$behaviors'] = [ new behaviors.CursorSync({ sync: oldModel.graphTooltip, }), queryController, + interactionTracker, registerDashboardMacro, registerPanelInteractionsReporter, new behaviors.LiveNowTimer({ enabled: oldModel.liveNow }), diff --git a/public/app/features/dashboard/services/DashboardProfiler.ts b/public/app/features/dashboard/services/DashboardProfiler.ts index af198ab2750..1be6cfd1f1c 100644 --- a/public/app/features/dashboard/services/DashboardProfiler.ts +++ b/public/app/features/dashboard/services/DashboardProfiler.ts @@ -10,6 +10,26 @@ export function getDashboardSceneProfiler() { return dashboardSceneProfiler; } +export function getDashboardComponentInteractionCallback(uid: string, title: string) { + return (e: SceneInteractionProfileEvent) => { + const payload = { + duration: e.duration, + networkDuration: e.networkDuration, + startTs: e.startTs, + endTs: e.endTs, + timeSinceBoot: performance.measure('time_since_boot', 'frontend_boot_js_done_time_seconds').duration, + }; + + reportInteraction('dashboard_interaction', { + interactionType: e.origin, + uid, + ...payload, + }); + + logMeasurement(`dashboard_interaction`, payload, { interactionType: e.origin, dashboard: uid, title: title }); + }; +} + export function getDashboardInteractionCallback(uid: string, title: string) { return (e: SceneInteractionProfileEvent) => { const payload = { diff --git a/public/app/features/scopes/selector/ScopesSelectorService.ts b/public/app/features/scopes/selector/ScopesSelectorService.ts index 5d8bf456f6a..92b00d16d96 100644 --- a/public/app/features/scopes/selector/ScopesSelectorService.ts +++ b/public/app/features/scopes/selector/ScopesSelectorService.ts @@ -1,4 +1,7 @@ import { Scope, ScopeNode, store as storeImpl } from '@grafana/data'; +import { config } from '@grafana/runtime'; +import { SceneRenderProfiler } from '@grafana/scenes'; +import { getDashboardSceneProfiler } from 'app/features/dashboard/services/DashboardProfiler'; import { ScopesApiClient } from '../ScopesApiClient'; import { ScopesServiceBase } from '../ScopesServiceBase'; @@ -49,7 +52,10 @@ export class ScopesSelectorService extends ScopesServiceBase { + this.interactionProfiler?.startInteraction('scopeNodeDiscovery'); + const path = getPathOfNode(scopeNodeId, this.state.nodes); const nodeToExpand = treeNodeAtPath(this.state.tree!, path); - if (!nodeToExpand) { - throw new Error(`Node ${scopeNodeId} not found in tree`); - } + try { + if (!nodeToExpand) { + throw new Error(`Node ${scopeNodeId} not found in tree`); + } - if (nodeToExpand.scopeNodeId !== '' && !isNodeExpandable(this.state.nodes[nodeToExpand.scopeNodeId])) { - throw new Error(`Trying to expand node at id ${scopeNodeId} that is not expandable`); - } + if (nodeToExpand.scopeNodeId !== '' && !isNodeExpandable(this.state.nodes[nodeToExpand.scopeNodeId])) { + throw new Error(`Trying to expand node at id ${scopeNodeId} that is not expandable`); + } - // Check if this is first expansion or filtering within existing children - const haveChildrenLoaded = nodeToExpand.children && Object.keys(nodeToExpand.children).length > 0; + // Check if this is first expansion or filtering within existing children + const haveChildrenLoaded = nodeToExpand.children && Object.keys(nodeToExpand.children).length > 0; - if (!nodeToExpand.expanded || nodeToExpand.query !== query || !haveChildrenLoaded) { - const newTree = modifyTreeNodeAtPath(this.state.tree!, path, (treeNode) => { - treeNode.expanded = true; - // Reset query on first expansion, keep it only when filtering within existing children - treeNode.query = ''; - }); - this.updateState({ tree: newTree }); + if (!nodeToExpand.expanded || nodeToExpand.query !== query || !haveChildrenLoaded) { + const newTree = modifyTreeNodeAtPath(this.state.tree!, path, (treeNode) => { + treeNode.expanded = true; + // Reset query on first expansion, keep it only when filtering within existing children + treeNode.query = ''; + }); + this.updateState({ tree: newTree }); - // For API call: only pass query if filtering within existing children - const queryForAPI = haveChildrenLoaded ? query : query === '' ? '' : undefined; - await this.loadNodeChildren(path, nodeToExpand, queryForAPI, haveChildrenLoaded); + // For API call: only pass query if filtering within existing children + const queryForAPI = haveChildrenLoaded ? query : query === '' ? '' : undefined; + await this.loadNodeChildren(path, nodeToExpand, queryForAPI, haveChildrenLoaded); + } + } catch (error) { + throw error; + } finally { + this.interactionProfiler?.stopInteraction(); } }; From db61acf513fd4eee73172b5c33eb253877b600d7 Mon Sep 17 00:00:00 2001 From: mohammad-hamid Date: Fri, 19 Sep 2025 10:55:08 -0400 Subject: [PATCH 08/54] `grafana-iam`: prevent duplicate permissions upon ResourcePermission create (#111361) prevent duplicate permissions --- .../apis/iam/resourcepermission/sql.go | 10 + .../apis/iam/resourcepermission/sql_test.go | 179 ++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/pkg/registry/apis/iam/resourcepermission/sql.go b/pkg/registry/apis/iam/resourcepermission/sql.go index 034c438d2ae..d635ef2db09 100644 --- a/pkg/registry/apis/iam/resourcepermission/sql.go +++ b/pkg/registry/apis/iam/resourcepermission/sql.go @@ -516,6 +516,16 @@ func validateCreateAndUpdateInput(v0ResourcePerm *v0alpha1.ResourcePermission, g return fmt.Errorf("resource permission name does not match spec: %w", errInvalidSpec) } + // Check for duplicate entities (same kind and name should appear only once) + seen := make(map[string]bool) + for _, perm := range v0ResourcePerm.Spec.Permissions { + key := fmt.Sprintf("%s:%s", perm.Kind, perm.Name) + if seen[key] { + return fmt.Errorf("duplicate entity found: kind=%s, name=%s (each entity can only appear once per resource): %w", perm.Kind, perm.Name, errInvalidSpec) + } + seen[key] = true + } + return nil } diff --git a/pkg/registry/apis/iam/resourcepermission/sql_test.go b/pkg/registry/apis/iam/resourcepermission/sql_test.go index c5a1d8feef9..8ff91859d9e 100644 --- a/pkg/registry/apis/iam/resourcepermission/sql_test.go +++ b/pkg/registry/apis/iam/resourcepermission/sql_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/grafana/authlib/types" @@ -668,3 +669,181 @@ func (f *fakeIdentityStore) GetUserInternalID(ctx context.Context, ns types.Name } return &legacy.GetUserInternalIDResult{ID: id}, nil } + +func TestValidateCreateAndUpdateInput(t *testing.T) { + grn := &groupResourceName{ + Group: "dashboard.grafana.app", + Resource: "dashboards", + Name: "test-dashboard", + } + + t.Run("Should pass validation with valid permissions", func(t *testing.T) { + resourcePerm := &v0alpha1.ResourcePermission{ + Spec: v0alpha1.ResourcePermissionSpec{ + Resource: v0alpha1.ResourcePermissionspecResource{ + ApiGroup: "dashboard.grafana.app", + Resource: "dashboards", + Name: "test-dashboard", + }, + Permissions: []v0alpha1.ResourcePermissionspecPermission{ + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "Editor", + Verb: "edit", + }, + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "Viewer", // Different entity name - should be allowed + Verb: "view", + }, + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindUser, + Name: "user-1", + Verb: "edit", // Different kind - should be allowed + }, + }, + }, + } + + err := validateCreateAndUpdateInput(resourcePerm, grn) + require.NoError(t, err) + }) + + t.Run("Should fail validation with duplicate entities", func(t *testing.T) { + resourcePerm := &v0alpha1.ResourcePermission{ + Spec: v0alpha1.ResourcePermissionSpec{ + Resource: v0alpha1.ResourcePermissionspecResource{ + ApiGroup: "dashboard.grafana.app", + Resource: "dashboards", + Name: "test-dashboard", + }, + Permissions: []v0alpha1.ResourcePermissionspecPermission{ + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "Editor", + Verb: "edit", + }, + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "Editor", + Verb: "view", // Same entity, different verb - should fail + }, + }, + }, + } + + err := validateCreateAndUpdateInput(resourcePerm, grn) + require.Error(t, err) + require.Contains(t, err.Error(), "duplicate entity found") + require.Contains(t, err.Error(), "kind=BasicRole") + require.Contains(t, err.Error(), "name=Editor") + require.Contains(t, err.Error(), "each entity can only appear once per resource") + }) + + t.Run("Should pass validation with same name but different kinds", func(t *testing.T) { + resourcePerm := &v0alpha1.ResourcePermission{ + Spec: v0alpha1.ResourcePermissionSpec{ + Resource: v0alpha1.ResourcePermissionspecResource{ + ApiGroup: "dashboard.grafana.app", + Resource: "dashboards", + Name: "test-dashboard", + }, + Permissions: []v0alpha1.ResourcePermissionspecPermission{ + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindUser, + Name: "editor", // Same name but different kind + Verb: "edit", + }, + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "editor", // Same name but different kind + Verb: "view", + }, + }, + }, + } + + err := validateCreateAndUpdateInput(resourcePerm, grn) + require.NoError(t, err) + }) +} + +func TestIntegration_UpdateResourcePermission_VerbChange(t *testing.T) { + testutil.SkipIntegrationTestInShortMode(t) + + backend := setupBackend(t) + backend.identityStore = NewFakeIdentityStore(t) + ctx := context.Background() + sql, err := backend.dbProvider(ctx) + require.NoError(t, err) + setupTestRoles(t, sql.DB) + + mapper := backend.mappers[schema.GroupResource{Group: "dashboard.grafana.app", Resource: "dashboards"}] + grn := &groupResourceName{Group: "dashboard.grafana.app", Resource: "dashboards", Name: "test-dash"} + + t.Run("should allow changing verb for same entity", func(t *testing.T) { + //Create initial permission with BasicRole Editor having "edit" verb + initialResourcePerm := &v0alpha1.ResourcePermission{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard.grafana.app-dashboards-test-dash", + Namespace: "default", + }, + Spec: v0alpha1.ResourcePermissionSpec{ + Resource: v0alpha1.ResourcePermissionspecResource{ + ApiGroup: "dashboard.grafana.app", + Resource: "dashboards", + Name: "test-dash", + }, + Permissions: []v0alpha1.ResourcePermissionspecPermission{ + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "Editor", + Verb: "edit", + }, + }, + }, + } + + rv, err := backend.createResourcePermission(ctx, sql, types.NamespaceInfo{Value: "default", OrgID: 1}, mapper, grn, initialResourcePerm) + require.NoError(t, err) + require.Greater(t, rv, int64(0)) + + //Update the same entity (BasicRole Editor) to have "view" verb instead + updatedResourcePerm := &v0alpha1.ResourcePermission{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dashboard.grafana.app-dashboards-test-dash", + Namespace: "default", + }, + Spec: v0alpha1.ResourcePermissionSpec{ + Resource: v0alpha1.ResourcePermissionspecResource{ + ApiGroup: "dashboard.grafana.app", + Resource: "dashboards", + Name: "test-dash", + }, + Permissions: []v0alpha1.ResourcePermissionspecPermission{ + { + Kind: v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, + Name: "Editor", + Verb: "view", + }, + }, + }, + } + + rv, err = backend.updateResourcePermission(ctx, sql, types.NamespaceInfo{Value: "default", OrgID: 1}, mapper, grn, updatedResourcePerm) + require.NoError(t, err) + require.Greater(t, rv, int64(0)) + + //Verify the update worked by reading back the permission + err = sql.DB.GetSqlxSession().WithTransaction(ctx, func(tx *session.SessionTx) error { + finalResourcePerm, err := backend.getResourcePermission(ctx, sql, tx, types.NamespaceInfo{Value: "default", OrgID: 1}, grn.string()) + require.NoError(t, err) + require.Len(t, finalResourcePerm.Spec.Permissions, 1) + require.Equal(t, v0alpha1.ResourcePermissionSpecPermissionKindBasicRole, finalResourcePerm.Spec.Permissions[0].Kind) + require.Equal(t, "Editor", finalResourcePerm.Spec.Permissions[0].Name) + require.Equal(t, "view", finalResourcePerm.Spec.Permissions[0].Verb) + return nil + }) + require.NoError(t, err) + }) +} From 1ef27e97499d1c3757051cfe36dbcd318743b3b6 Mon Sep 17 00:00:00 2001 From: colin-stuart Date: Fri, 19 Sep 2025 09:55:18 -0500 Subject: [PATCH 09/54] Auth: Add SCIM settings permission to auth config writer role (#111326) * Auth: add SCIM settings permission to authentication config writer role * make update-workspace --- pkg/services/accesscontrol/models.go | 1 + pkg/services/accesscontrol/roles.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/pkg/services/accesscontrol/models.go b/pkg/services/accesscontrol/models.go index 8cfb9d5121b..ad6894feb44 100644 --- a/pkg/services/accesscontrol/models.go +++ b/pkg/services/accesscontrol/models.go @@ -392,6 +392,7 @@ const ( // Settings scope ScopeSettingsAll = "settings:*" ScopeSettingsSAML = "settings:auth.saml:*" + ScopeSettingsSCIM = "settings:auth.scim:*" // Team related actions ActionTeamsCreate = "teams:create" diff --git a/pkg/services/accesscontrol/roles.go b/pkg/services/accesscontrol/roles.go index 6ed0357d9de..6d01e3b8624 100644 --- a/pkg/services/accesscontrol/roles.go +++ b/pkg/services/accesscontrol/roles.go @@ -273,6 +273,14 @@ var ( Action: ActionSettingsWrite, Scope: ScopeSettingsOAuth("ldap"), }, + { + Action: ActionSettingsRead, + Scope: ScopeSettingsSCIM, + }, + { + Action: ActionSettingsWrite, + Scope: ScopeSettingsSCIM, + }, }, } From 6203a6f3c55b0676fd0b485e0c58fd688dbe04a8 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Fri, 19 Sep 2025 08:58:36 -0600 Subject: [PATCH 10/54] Provisioning: Allow non-loopback config (#111353) --- pkg/registry/apis/provisioning/register.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index cd353ba5ccd..c0fdc90ca34 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -22,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" + clientrest "k8s.io/client-go/rest" "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" "k8s.io/kube-openapi/pkg/validation/spec" @@ -114,6 +115,8 @@ type APIBuilder struct { // Extras provides additional functionality to the API. extras []Extra extraWorkers []jobs.Worker + + restConfigGetter func(context.Context) (*clientrest.Config, error) } // NewAPIBuilder creates an API builder. @@ -134,6 +137,7 @@ func NewAPIBuilder( extraWorkers []jobs.Worker, jobHistoryConfig *JobHistoryConfig, allowedTargets []provisioning.SyncTargetType, + restConfigGetter func(context.Context) (*clientrest.Config, error), allowImageRendering bool, newStandaloneClientFactoryFunc func(loopbackConfigProvider apiserver.RestConfigProvider) resources.ClientFactory, // optional, only used for standalone apiserver ) *APIBuilder { @@ -163,6 +167,7 @@ func NewAPIBuilder( jobHistoryConfig: jobHistoryConfig, extraWorkers: extraWorkers, allowedTargets: allowedTargets, + restConfigGetter: restConfigGetter, allowImageRendering: allowImageRendering, } @@ -247,6 +252,7 @@ func RegisterAPIService( extraWorkers, createJobHistoryConfigFromSettings(cfg), allowedTargets, + nil, // will use loopback instead cfg.ProvisioningAllowImageRendering, nil, ) @@ -678,7 +684,17 @@ func (b *APIBuilder) verifyAgainstExistingRepositories(cfg *provisioning.Reposit func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartHookFunc, error) { postStartHooks := map[string]genericapiserver.PostStartHookFunc{ "grafana-provisioning": func(postStartHookCtx genericapiserver.PostStartHookContext) error { - c, err := clientset.NewForConfig(postStartHookCtx.LoopbackClientConfig) + var config *clientrest.Config + var err error + if b.restConfigGetter == nil { + config = postStartHookCtx.LoopbackClientConfig + } else { + config, err = b.restConfigGetter(postStartHookCtx.Context) + if err != nil { + return err + } + } + c, err := clientset.NewForConfig(config) if err != nil { return err } From c0ce4ff1f2f265730af01b01fe37f67c239fca63 Mon Sep 17 00:00:00 2001 From: Ivan Ortega Alba Date: Fri, 19 Sep 2025 18:03:02 +0200 Subject: [PATCH 11/54] Dashboards: Fix missing Ctrl+O keyboard shortcut for crosshair toggle (#111310) * Dashboard Scenes: Fix missing Ctrl+O keyboard shortcut for crosshair toggle - Add missing mod+o keybind to dashboard scenes keyboard shortcuts - Implement crosshair state cycling (Default -> Crosshair -> Tooltip -> Default) - Add comprehensive unit tests for keyboard shortcuts functionality - Add e2e test to verify shortcut works and prevents browser file dialog - Fix ensures parity between legacy and scenes dashboard implementations Fixes issue where Ctrl+O/Cmd+O was opening browser file dialog instead of toggling shared crosshair modes in scenes-based dashboards. * Remove waitForTimeout from e2e test - Replace arbitrary timeouts with proper element waiting - Use waitFor with visible state instead of setTimeout - Improve test reliability and follow Playwright best practices * Optimize e2e test for crosshair keyboard shortcut - Remove unnecessary console logging and timeout settings - Simplify assertions to only check the currently selected radio button - Improve test performance by reducing DOM queries - Focus on essential functionality verification * Fix linting --- e2e-playwright/various-suite/keybinds.spec.ts | 58 ++++ go.work.sum | 1 + .../scene/keyboardShortcuts.test.ts | 254 ++++++++++++++++++ .../scene/keyboardShortcuts.ts | 18 +- 4 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 public/app/features/dashboard-scene/scene/keyboardShortcuts.test.ts diff --git a/e2e-playwright/various-suite/keybinds.spec.ts b/e2e-playwright/various-suite/keybinds.spec.ts index 4ff37889838..74549a620e8 100644 --- a/e2e-playwright/various-suite/keybinds.spec.ts +++ b/e2e-playwright/various-suite/keybinds.spec.ts @@ -85,5 +85,63 @@ test.describe( expectedRange = 'Time range selected: 2024-06-05 10:04:00 to 2024-06-05 10:05:00'; // 1 min back await expect(timePickerButton).toHaveAttribute('aria-label', expectedRange); }); + + test('ctrl+o should toggle shared crosshair', async ({ page, selectors }) => { + // Navigate to a new dashboard + await page.goto('/dashboard/new?orgId=1'); + + // Wait for dashboard to load + await page.waitForLoadState('networkidle'); + + // Wait for dashboard to be fully initialized by checking for dashboard content + await page + .locator('[data-testid*="dashboard"]') + .or(page.locator('text=Start your new dashboard')) + .first() + .waitFor({ state: 'visible' }); + + // Test the keyboard shortcut first in the main dashboard view + const currentUrl = page.url(); + const modKey = process.platform === 'darwin' ? 'Meta' : 'Control'; + + // Test that mod+o works in the main dashboard (should not trigger file dialog) + console.log('Testing mod+o in main dashboard view...'); + await page.keyboard.press(`${modKey}+o`); + expect(page.url()).toBe(currentUrl); // Should not navigate away + + // Now open settings to check if the state actually changed + await page.keyboard.press('d'); + await page.keyboard.press('s'); + + // Wait for settings page to load by checking for the General tab or settings content + await page + .locator('text=General') + .or(page.locator('[data-testid*="dashboard-settings"]')) + .waitFor({ state: 'visible' }); + + // Wait for Panel options section to be visible and scroll to it + const panelOptionsSection = page.locator('text=Panel options'); + await panelOptionsSection.waitFor({ state: 'visible' }); + await panelOptionsSection.scrollIntoViewIfNeeded(); + + // Wait for radio buttons to be visible + await page + .locator('[role="radiogroup"]') + .last() + .locator('input[type="radio"]') + .first() + .waitFor({ state: 'visible' }); + + // Check current state - after one mod+o press, it should be crosshair (1) + await expect(page.locator('[role="radiogroup"]').last().locator('input[type="radio"]').nth(1)).toBeChecked(); // Shared crosshair + + // Test second press in the main dashboard view (should go to tooltip) + await page.keyboard.press(`${modKey}+o`); + await expect(page.locator('[role="radiogroup"]').last().locator('input[type="radio"]').nth(2)).toBeChecked(); // Shared tooltip + + // Test third press in the main dashboard view (should go back to default) + await page.keyboard.press(`${modKey}+o`); + await expect(page.locator('[role="radiogroup"]').last().locator('input[type="radio"]').nth(0)).toBeChecked(); // Default + }); } ); diff --git a/go.work.sum b/go.work.sum index 5d029982567..6dd02118ee9 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1055,6 +1055,7 @@ github.com/grafana/dskit v0.0.0-20250818234656-8ff9c6532e85/go.mod h1:kImsvJ1xnm github.com/grafana/go-gelf/v2 v2.0.1 h1:BOChP0h/jLeD+7F9mL7tq10xVkDG15he3T1zHuQaWak= github.com/grafana/go-gelf/v2 v2.0.1/go.mod h1:lexHie0xzYGwCgiRGcvZ723bSNyNI8ZRD4s0CLobh90= github.com/grafana/gomemcache v0.0.0-20250228145437-da7b95fd2ac1/go.mod h1:j/s0jkda4UXTemDs7Pgw/vMT06alWc42CHisvYac0qw= +github.com/grafana/gomemcache v0.0.0-20250828162811-a96f6acee2fe/go.mod h1:j/s0jkda4UXTemDs7Pgw/vMT06alWc42CHisvYac0qw= github.com/grafana/grafana-app-sdk v0.40.1/go.mod h1:4P8h7VB6KcDjX9bAoBQc6IP8iNylxe6bSXLR9gA39gM= github.com/grafana/grafana-app-sdk v0.41.0 h1:SYHN3U7B1myRKY3UZZDkFsue9TDmAOap0UrQVTqtYBU= github.com/grafana/grafana-app-sdk v0.41.0/go.mod h1:Wg/3vEZfok1hhIWiHaaJm+FwkosfO98o8KbeLFEnZpY= diff --git a/public/app/features/dashboard-scene/scene/keyboardShortcuts.test.ts b/public/app/features/dashboard-scene/scene/keyboardShortcuts.test.ts new file mode 100644 index 00000000000..27573069fa3 --- /dev/null +++ b/public/app/features/dashboard-scene/scene/keyboardShortcuts.test.ts @@ -0,0 +1,254 @@ +import { LegacyGraphHoverClearEvent } from '@grafana/data'; +import { behaviors, sceneGraph, SceneTimeRange } from '@grafana/scenes'; +import { DashboardCursorSync } from '@grafana/schema'; +import appEvents from 'app/core/app_events'; +import { KeybindingSet } from 'app/core/services/KeybindingSet'; + +import { DashboardScene } from './DashboardScene'; +import { setupKeyboardShortcuts } from './keyboardShortcuts'; + +// Mock dependencies +jest.mock('app/core/app_events', () => ({ + subscribe: jest.fn(() => ({ unsubscribe: jest.fn() })), + publish: jest.fn(), +})); +jest.mock('app/core/services/KeybindingSet'); +const mockOnRefresh = jest.fn(); +jest.mock('@grafana/scenes', () => ({ + ...jest.requireActual('@grafana/scenes'), + sceneGraph: { + getTimeRange: jest.fn(() => ({ + onRefresh: mockOnRefresh, + })), + }, +})); + +describe('setupKeyboardShortcuts', () => { + let mockScene: DashboardScene; + let mockKeybindingSet: jest.Mocked; + let mockCursorSync: behaviors.CursorSync; + + beforeEach(() => { + jest.clearAllMocks(); + mockOnRefresh.mockClear(); + + // Mock KeybindingSet + mockKeybindingSet = jest.mocked(new KeybindingSet()); + jest.spyOn(mockKeybindingSet, 'addBinding').mockImplementation(); + jest.spyOn(mockKeybindingSet, 'removeAll').mockImplementation(); + (KeybindingSet as jest.Mock).mockImplementation(() => mockKeybindingSet); + + // Create mock CursorSync behavior + mockCursorSync = new behaviors.CursorSync({ sync: DashboardCursorSync.Off }); + jest.spyOn(mockCursorSync, 'setState'); + + // Create mock DashboardScene + mockScene = new DashboardScene({ + title: 'Test Dashboard', + uid: 'test-uid', + $timeRange: new SceneTimeRange({ from: 'now-6h', to: 'now' }), + $behaviors: [mockCursorSync], + }); + + // Mock canEditDashboard + jest.spyOn(mockScene, 'canEditDashboard').mockReturnValue(true); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should setup keyboard shortcuts and return cleanup function', () => { + const cleanup = setupKeyboardShortcuts(mockScene); + + expect(KeybindingSet).toHaveBeenCalled(); + expect(mockKeybindingSet.addBinding).toHaveBeenCalled(); + expect(typeof cleanup).toBe('function'); + + // Call cleanup function + cleanup(); + expect(mockKeybindingSet.removeAll).toHaveBeenCalled(); + }); + + describe('mod+o shortcut (toggle shared crosshair)', () => { + let modOHandler: () => void; + + beforeEach(() => { + setupKeyboardShortcuts(mockScene); + + // Find the mod+o binding + const modOBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'mod+o'); + expect(modOBinding).toBeDefined(); + modOHandler = modOBinding![0].onTrigger; + }); + + it('should toggle cursor sync from Off to Crosshair', () => { + // Initial state: Off (0) + mockCursorSync.setState({ sync: DashboardCursorSync.Off }); + + modOHandler(); + + expect(mockCursorSync.setState).toHaveBeenCalledWith({ sync: DashboardCursorSync.Crosshair }); + expect(appEvents.publish).toHaveBeenCalledWith(expect.any(LegacyGraphHoverClearEvent)); + expect(sceneGraph.getTimeRange).toHaveBeenCalledWith(mockScene); + expect(mockOnRefresh).toHaveBeenCalled(); + }); + + it('should toggle cursor sync from Crosshair to Tooltip', () => { + // Initial state: Crosshair (1) + mockCursorSync.setState({ sync: DashboardCursorSync.Crosshair }); + jest.clearAllMocks(); + + modOHandler(); + + expect(mockCursorSync.setState).toHaveBeenCalledWith({ sync: DashboardCursorSync.Tooltip }); + expect(appEvents.publish).toHaveBeenCalledWith(expect.any(LegacyGraphHoverClearEvent)); + expect(sceneGraph.getTimeRange).toHaveBeenCalledWith(mockScene); + expect(mockOnRefresh).toHaveBeenCalled(); + }); + + it('should toggle cursor sync from Tooltip to Off', () => { + // Initial state: Tooltip (2) + mockCursorSync.setState({ sync: DashboardCursorSync.Tooltip }); + jest.clearAllMocks(); + + modOHandler(); + + expect(mockCursorSync.setState).toHaveBeenCalledWith({ sync: DashboardCursorSync.Off }); + expect(appEvents.publish).toHaveBeenCalledWith(expect.any(LegacyGraphHoverClearEvent)); + expect(sceneGraph.getTimeRange).toHaveBeenCalledWith(mockScene); + expect(mockOnRefresh).toHaveBeenCalled(); + }); + + it('should handle missing CursorSync behavior gracefully', () => { + // Create scene without CursorSync behavior by overriding state + const sceneWithoutCursorSync = new DashboardScene({ + title: 'Test Dashboard', + uid: 'test-uid', + $timeRange: new SceneTimeRange({ from: 'now-6h', to: 'now' }), + }); + + // Force remove CursorSync behavior + sceneWithoutCursorSync.setState({ $behaviors: [] }); + jest.clearAllMocks(); + + setupKeyboardShortcuts(sceneWithoutCursorSync); + + const modOBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'mod+o'); + const handler = modOBinding![0].onTrigger; + + // Should not throw error when CursorSync is missing + expect(() => handler()).not.toThrow(); + // When CursorSync is missing, no state changes or refresh should happen + expect(sceneGraph.getTimeRange).not.toHaveBeenCalled(); + }); + + it('should handle non-CursorSync behavior gracefully', () => { + // Create scene with different behavior type + const sceneWithOtherBehavior = new DashboardScene({ + title: 'Test Dashboard', + uid: 'test-uid', + $timeRange: new SceneTimeRange({ from: 'now-6h', to: 'now' }), + }); + + // Force set only non-CursorSync behavior + sceneWithOtherBehavior.setState({ $behaviors: [new behaviors.LiveNowTimer({ enabled: false })] }); + jest.clearAllMocks(); + + setupKeyboardShortcuts(sceneWithOtherBehavior); + + const modOBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'mod+o'); + const handler = modOBinding![0].onTrigger; + + // Should not throw error when CursorSync is not found + expect(() => handler()).not.toThrow(); + // When CursorSync is not found, no state changes or refresh should happen + expect(sceneGraph.getTimeRange).not.toHaveBeenCalled(); + }); + }); + + describe('other keyboard shortcuts', () => { + beforeEach(() => { + setupKeyboardShortcuts(mockScene); + }); + + it('should setup view panel shortcut (v)', () => { + const vBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'v'); + expect(vBinding).toBeDefined(); + }); + + it('should setup refresh shortcut (d r)', () => { + const drBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'd r'); + expect(drBinding).toBeDefined(); + }); + + it('should setup zoom out shortcut (t z)', () => { + const tzBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 't z'); + expect(tzBinding).toBeDefined(); + }); + + it('should setup zoom out shortcut (ctrl+z)', () => { + const ctrlZBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'ctrl+z'); + expect(ctrlZBinding).toBeDefined(); + }); + + it('should setup time range shortcuts', () => { + const taBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 't a'); + const tLeftBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 't left'); + const tRightBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 't right'); + + expect(taBinding).toBeDefined(); + expect(tLeftBinding).toBeDefined(); + expect(tRightBinding).toBeDefined(); + }); + }); + + describe('edit mode shortcuts', () => { + beforeEach(() => { + jest.spyOn(mockScene, 'canEditDashboard').mockReturnValue(true); + setupKeyboardShortcuts(mockScene); + }); + + it('should setup edit panel shortcut (e) when can edit', () => { + const eBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'e'); + expect(eBinding).toBeDefined(); + }); + + it('should setup save shortcut (mod+s) when can edit', () => { + const modSBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'mod+s'); + expect(modSBinding).toBeDefined(); + }); + + it('should setup dashboard settings shortcut (d s) when can edit', () => { + const dsBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'd s'); + expect(dsBinding).toBeDefined(); + }); + }); + + describe('non-edit mode', () => { + beforeEach(() => { + jest.spyOn(mockScene, 'canEditDashboard').mockReturnValue(false); + setupKeyboardShortcuts(mockScene); + }); + + it('should not setup edit-only shortcuts when cannot edit', () => { + const eBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'e'); + const modSBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'mod+s'); + const dsBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'd s'); + + expect(eBinding).toBeUndefined(); + expect(modSBinding).toBeUndefined(); + expect(dsBinding).toBeUndefined(); + }); + + it('should still setup non-edit shortcuts when cannot edit', () => { + const modOBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'mod+o'); + const vBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'v'); + const drBinding = mockKeybindingSet.addBinding.mock.calls.find((call) => call[0].key === 'd r'); + + expect(modOBinding).toBeDefined(); + expect(vBinding).toBeDefined(); + expect(drBinding).toBeDefined(); + }); + }); +}); diff --git a/public/app/features/dashboard-scene/scene/keyboardShortcuts.ts b/public/app/features/dashboard-scene/scene/keyboardShortcuts.ts index 61c42b6ceeb..0da6f08c2bc 100644 --- a/public/app/features/dashboard-scene/scene/keyboardShortcuts.ts +++ b/public/app/features/dashboard-scene/scene/keyboardShortcuts.ts @@ -1,6 +1,6 @@ -import { locationUtil, SetPanelAttentionEvent } from '@grafana/data'; +import { locationUtil, SetPanelAttentionEvent, LegacyGraphHoverClearEvent } from '@grafana/data'; import { config, locationService } from '@grafana/runtime'; -import { sceneGraph, VizPanel } from '@grafana/scenes'; +import { behaviors, sceneGraph, VizPanel } from '@grafana/scenes'; import appEvents from 'app/core/app_events'; import { KeybindingSet } from 'app/core/services/KeybindingSet'; import { contextSrv } from 'app/core/services/context_srv'; @@ -168,6 +168,20 @@ export function setupKeyboardShortcuts(scene: DashboardScene) { }, }); + keybindings.addBinding({ + key: 'mod+o', + onTrigger: () => { + const cursorSync = scene.state.$behaviors?.find((b) => b instanceof behaviors.CursorSync); + if (cursorSync instanceof behaviors.CursorSync) { + const currentSync = cursorSync.state.sync; + const nextSync = (currentSync + 1) % 3; + cursorSync.setState({ sync: nextSync }); + appEvents.publish(new LegacyGraphHoverClearEvent()); + sceneGraph.getTimeRange(scene).onRefresh(); + } + }, + }); + if (canEdit) { // Panel edit keybindings.addBinding({ From 316b5fe53872c2e7a325adb20b0b91e3b1d99582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Jamr=C3=B3z?= Date: Fri, 19 Sep 2025 18:12:41 +0200 Subject: [PATCH 12/54] Explore: Disable compact mode for ClickHouse plugin (#110869) --- public/app/features/explore/Explore.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index a895750bb59..f9660d5833a 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -312,7 +312,19 @@ export class Explore extends PureComponent { */ onSplitOpen = (panelType: string) => { return async (options?: SplitOpenOptions) => { - this.props.splitOpen(options ? { ...options, compact: true } : options); + let compact = true; + + /** + * Temporary fix grafana-clickhouse-datasource as it requires the query editor to be fully rendered to update the query + * Proposed fixes: + * - https://github.com/grafana/clickhouse-datasource/issues/1363 - handle query update in data source + * - https://github.com/grafana/grafana/issues/110868 - allow data links to provide meta info if the link can be handled in compact mode (default to false) + */ + if (options?.queries?.some((q) => q.datasource?.type === 'grafana-clickhouse-datasource')) { + compact = false; + } + + this.props.splitOpen(options ? { ...options, compact } : options); if (options && this.props.datasourceInstance) { const target = (await getDataSourceSrv().get(options.datasourceUid)).type; const source = From b3ed600bca1272a503e344a4c3d09a8ae7bac6f1 Mon Sep 17 00:00:00 2001 From: maicon Date: Fri, 19 Sep 2025 15:03:23 -0300 Subject: [PATCH 13/54] Revert "Unistore: Add validation for resource names" (#111408) Revert "Unistore: Add validation for resource names (#110990)" This reverts commit b63e3fd3aebb73fa1d0b6c1f97334660e18a2a25. --- .../storage/testing/watcher_tests.go | 8 +- pkg/storage/unified/resource/bulk.go | 12 -- pkg/storage/unified/resource/keys.go | 12 -- pkg/storage/unified/resource/keys_test.go | 115 ---------- pkg/storage/unified/resource/server.go | 4 - pkg/storage/unified/resource/server_test.go | 197 ------------------ pkg/storage/unified/resource/validation.go | 20 -- 7 files changed, 4 insertions(+), 364 deletions(-) diff --git a/pkg/apiserver/storage/testing/watcher_tests.go b/pkg/apiserver/storage/testing/watcher_tests.go index 371a3d612f9..89ea37d4b95 100644 --- a/pkg/apiserver/storage/testing/watcher_tests.go +++ b/pkg/apiserver/storage/testing/watcher_tests.go @@ -684,7 +684,7 @@ func RunTestClusterScopedWatch(ctx context.Context, t *testing.T, store storage. currentObjs := map[string]*example.Pod{} for _, watchTest := range tt.watchTests { out := &example.Pod{} - key := "pods/ns-1/" + watchTest.obj.Name + key := "pods/" + watchTest.obj.Name err := store.GuaranteedUpdate(ctx, key, out, true, nil, storage.SimpleUpdate( func(runtime.Object) (runtime.Object, error) { obj := watchTest.obj.DeepCopy() @@ -1607,15 +1607,15 @@ func clusterScopedNodeNameAttrFunc(obj runtime.Object) (labels.Set, fields.Set, } func basePod(podName string) *example.Pod { - return baseNamespacedPod(podName, "ns-1") + return baseNamespacedPod(podName, "") } func basePodUpdated(podName string) *example.Pod { - return baseNamespacedPodUpdated(podName, "ns-1") + return baseNamespacedPodUpdated(podName, "") } func basePodAssigned(podName, nodeName string) *example.Pod { - return baseNamespacedPodAssigned(podName, "ns-1", nodeName) + return baseNamespacedPodAssigned(podName, "", nodeName) } func baseNamespacedPod(podName, namespace string) *example.Pod { diff --git a/pkg/storage/unified/resource/bulk.go b/pkg/storage/unified/resource/bulk.go index a0720efa6de..90298c0da66 100644 --- a/pkg/storage/unified/resource/bulk.go +++ b/pkg/storage/unified/resource/bulk.go @@ -170,18 +170,6 @@ func (s *server) BulkProcess(stream resourcepb.BulkStore_BulkProcessServer) erro }) } - // Verify all request keys are valid - for _, k := range settings.Collection { - if r := verifyRequestKey(k); r != nil { - return sendAndClose(&resourcepb.BulkResponse{ - Error: &resourcepb.ErrorResult{ - Message: fmt.Sprintf("invalid request key: %s", r.Message), - Code: http.StatusBadRequest, - }, - }) - } - } - if settings.RebuildCollection { for _, k := range settings.Collection { // Can we delete the whole collection diff --git a/pkg/storage/unified/resource/keys.go b/pkg/storage/unified/resource/keys.go index 204246f3497..0af18d6c048 100644 --- a/pkg/storage/unified/resource/keys.go +++ b/pkg/storage/unified/resource/keys.go @@ -17,18 +17,6 @@ func verifyRequestKey(key *resourcepb.ResourceKey) *resourcepb.ErrorResult { if key.Resource == "" { return NewBadRequestError("request key is missing resource") } - if err := validateName(key.Name); err != nil { - return NewBadRequestError(fmt.Sprintf("name '%s' is invalid: '%s'", key.Name, err)) - } - if err := validateQualifiedName(key.Namespace); err != nil { - return NewBadRequestError(fmt.Sprintf("namespace '%s' is invalid: '%s'", key.Namespace, err)) - } - if err := validateQualifiedName(key.Group); err != nil { - return NewBadRequestError(fmt.Sprintf("group '%s' is invalid: '%s'", key.Group, err)) - } - if err := validateQualifiedName(key.Resource); err != nil { - return NewBadRequestError(fmt.Sprintf("resource '%s' is invalid: '%s'", key.Resource, err)) - } return nil } diff --git a/pkg/storage/unified/resource/keys_test.go b/pkg/storage/unified/resource/keys_test.go index f86f1224313..d14c03fcc95 100644 --- a/pkg/storage/unified/resource/keys_test.go +++ b/pkg/storage/unified/resource/keys_test.go @@ -1,8 +1,6 @@ package resource import ( - "net/http" - "strings" "testing" "github.com/stretchr/testify/require" @@ -68,116 +66,3 @@ func TestSearchIDKeys(t *testing.T) { } } } - -func TestVerifyRequestKey(t *testing.T) { - validGroup := "group.grafana.app" - validResource := "resource" - validNamespace := "default" - validName := "fdgsv37qslr0ga" - validLegacyUID := "f8cc010c-ee72-4681-89d2-d46e1bd47d33" - - invalidGroup := "group.~~~~~grafana.app" - invalidResource := "##resource" - invalidNamespace := "(((((default" - invalidName := " " // only spaces - - namespaceTooLong := strings.Repeat("a", MaxQualifiedNameLength+1) - nameTooLong := strings.Repeat("a", 300) - - tests := []struct { - name string - input *resourcepb.ResourceKey - expectedCode int32 - }{ - { - name: "no error when all fields are set and valid", - input: &resourcepb.ResourceKey{ - Namespace: validNamespace, - Group: validGroup, - Resource: validResource, - Name: validName, - }, - }, - { - name: "invalid namespace returns error", - input: &resourcepb.ResourceKey{ - Namespace: invalidNamespace, - Group: validGroup, - Resource: validResource, - Name: validName, - }, - expectedCode: http.StatusBadRequest, - }, - { - name: "invalid group returns error", - input: &resourcepb.ResourceKey{ - Namespace: validNamespace, - Group: invalidGroup, - Resource: validResource, - Name: validName, - }, - expectedCode: http.StatusBadRequest, - }, - { - name: "invalid resource returns error", - input: &resourcepb.ResourceKey{ - Namespace: validNamespace, - Group: validGroup, - Resource: invalidResource, - Name: validName, - }, - expectedCode: http.StatusBadRequest, - }, - { - name: "invalid name returns error", - input: &resourcepb.ResourceKey{ - Namespace: validNamespace, - Group: validGroup, - Resource: validResource, - Name: invalidName, - }, - expectedCode: http.StatusBadRequest, - }, - { - name: "valid legacy UID returns no error", - input: &resourcepb.ResourceKey{ - Namespace: validNamespace, - Group: validGroup, - Resource: validResource, - Name: validLegacyUID, - }, - }, - { - name: "namespace too long returns error", - input: &resourcepb.ResourceKey{ - Namespace: namespaceTooLong, - Group: validGroup, - Resource: validResource, - Name: validName, - }, - expectedCode: http.StatusBadRequest, - }, - { - name: "name too long returns error", - input: &resourcepb.ResourceKey{ - Namespace: namespaceTooLong, - Group: validGroup, - Resource: validResource, - Name: nameTooLong, - }, - expectedCode: http.StatusBadRequest, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := verifyRequestKey(test.input) - if test.expectedCode == 0 { - require.Nil(t, err) - return - } - - require.Equal(t, test.expectedCode, err.Code) - }) - } -} diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index 55fe947fa56..35987a418b8 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -626,10 +626,6 @@ func (s *server) Create(ctx context.Context, req *resourcepb.CreateRequest) (*re ctx, span := s.tracer.Start(ctx, "storage_server.Create") defer span.End() - if r := verifyRequestKey(req.Key); r != nil { - return nil, fmt.Errorf("invalid request key: %s", r.Message) - } - rsp := &resourcepb.CreateResponse{} user, ok := claims.AuthInfoFrom(ctx) if !ok || user == nil { diff --git a/pkg/storage/unified/resource/server_test.go b/pkg/storage/unified/resource/server_test.go index 150ee328af1..98bc928da63 100644 --- a/pkg/storage/unified/resource/server_test.go +++ b/pkg/storage/unified/resource/server_test.go @@ -7,7 +7,6 @@ import ( "log/slog" "net/http" "os" - "strings" "sync" "testing" "time" @@ -196,202 +195,6 @@ func TestSimpleServer(t *testing.T) { require.Len(t, all.Items, 0) // empty }) - t.Run("playlist FAIL CRUD paths due to invalid key", func(t *testing.T) { - raw := []byte(`{ - "apiVersion": "playlist.grafana.app/v0alpha1", - "kind": "Playlist", - "metadata": { - "name": "fdgsv37#qslr0ga", - "uid": "xyz", - "namespace": "default", - "annotations": { - "grafana.app/repoName": "elsewhere", - "grafana.app/repoPath": "path/to/item", - "grafana.app/repoTimestamp": "2024-02-02T00:00:00Z" - } - }, - "spec": { - "title": "hello", - "interval": "5m", - "items": [ - { - "type": "dashboard_by_uid", - "value": "vmie2cmWz" - } - ] - } - }`) - - // invalid group - key := &resourcepb.ResourceKey{ - Group: "playlist.grafana.app###", - Resource: "rrrr", // can be anything :( - Namespace: "default", - Name: "fdgsv37qslr0ga", - } - - created, err := server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - require.Error(t, err) - require.Nil(t, created) - - // invalid resource - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr###", // can be anything :( - Namespace: "default", - Name: "fdgsv37qslr0ga", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - require.Error(t, err) - require.Nil(t, created) - - // invalid namespace - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr", // can be anything :( - Namespace: "default###", - Name: "fdgsv37qslr0ga", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - require.Error(t, err) - require.Nil(t, created) - - // invalid name - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr", // can be anything :( - Namespace: "default", - Name: "fdgsv37qslr0g###", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - require.Error(t, err) - require.Nil(t, created) - - // legacy name - valid - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr", // can be anything :( - Namespace: "default", - Name: "2c7e5361-7360-4d2a-ae45-5e79bba458d6", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - - require.NoError(t, err) - require.NotNil(t, created) - - // legacy name - also valid - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr", // can be anything :( - Namespace: "default", - Name: "IvIsO_YGz", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - - require.NoError(t, err) - require.NotNil(t, created) - - // legacy name - also valid - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr", // can be anything :( - Namespace: "default", - Name: "_IvIsOYGz", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - - require.NoError(t, err) - require.NotNil(t, created) - - invalidQualifiedNames := []string{ - "", // empty - strings.Repeat("1", MaxQualifiedNameLength+1), // too long - " ", // only spaces - "f8cc010c.ee72.4681;89d2+d46e1bd47d33", // invalid chars - } - - // group - for _, invalidGroup := range invalidQualifiedNames { - key = &resourcepb.ResourceKey{ - Group: invalidGroup, - Resource: "rrrr", // can be anything :( - Namespace: "default", - Name: "_IvIsOYGz", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - - require.Error(t, err) - require.Nil(t, created) - } - - // resource - for _, invalidResource := range invalidQualifiedNames { - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: invalidResource, - Namespace: "default", - Name: "_IvIsOYGz", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - - require.Error(t, err) - require.Nil(t, created) - } - - // namespace - for _, invalidNamespace := range invalidQualifiedNames { - key = &resourcepb.ResourceKey{ - Group: "playlist.grafana.app", - Resource: "rrrr", // can be anything :( - Namespace: invalidNamespace, - Name: "_IvIsOYGz", - } - - created, err = server.Create(ctx, &resourcepb.CreateRequest{ - Value: raw, - Key: key, - }) - - require.Error(t, err) - require.Nil(t, created) - } - }) - t.Run("playlist update optimistic concurrency check", func(t *testing.T) { raw := []byte(`{ "apiVersion": "playlist.grafana.app/v0alpha1", diff --git a/pkg/storage/unified/resource/validation.go b/pkg/storage/unified/resource/validation.go index c3d3ee5bd45..2c1b5cf3740 100644 --- a/pkg/storage/unified/resource/validation.go +++ b/pkg/storage/unified/resource/validation.go @@ -1,15 +1,11 @@ package resource import ( - "fmt" "regexp" "github.com/grafana/grafana/pkg/storage/unified/resourcepb" - "k8s.io/apimachinery/pkg/util/validation" ) -const MaxQualifiedNameLength = 40 - var validNameCharPattern = `a-zA-Z0-9:\-\_\.` var validNamePattern = regexp.MustCompile(`^[` + validNameCharPattern + `]*$`).MatchString @@ -28,19 +24,3 @@ func validateName(name string) *resourcepb.ErrorResult { // so we will be slightly more lenient than standard k8s return nil } - -func validateQualifiedName(value string) *resourcepb.ErrorResult { - if len(value) == 0 { - return NewBadRequestError("value is too short") - } - if len(value) > MaxQualifiedNameLength { - return NewBadRequestError("value is too long") - } - - err := validation.IsQualifiedName(value) - if len(err) > 0 { - return NewBadRequestError(fmt.Sprintf("name is not a valid qualified name: %+v", err)) - } - - return nil -} From e5121e8762379b2077228628a81e3241054beb95 Mon Sep 17 00:00:00 2001 From: Charandas <542168+charandas@users.noreply.github.com> Date: Fri, 19 Sep 2025 11:48:31 -0700 Subject: [PATCH 14/54] feat: customizable persistent volumes for devenv docker blocks (#111412) --- .github/CODEOWNERS | 1 + devenv/create_docker_compose.sh | 28 ++++++++++++++++++++++++ devenv/docker/compose_volume_section.yml | 1 + 3 files changed, 30 insertions(+) create mode 100644 devenv/docker/compose_volume_section.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c7339cdb3a9..df081020c75 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -310,6 +310,7 @@ /devenv/docker/blocks/webdav/ @grafana/alerting-backend /devenv/docker/buildcontainer/ @bergquist /devenv/docker/compose_header.yml @grafana/grafana-backend-services-squad +/devenv/docker/compose_volume_section.yml @grafana/grafana-backend-services-squad /devenv/docker/debtest/ @bergquist /devenv/docker/ha-test-unified-alerting/ @grafana/alerting-backend /devenv/docker/ha_test/ @grafana/grafana-backend-services-squad diff --git a/devenv/create_docker_compose.sh b/devenv/create_docker_compose.sh index d937f95cec4..baa50818f11 100755 --- a/devenv/create_docker_compose.sh +++ b/devenv/create_docker_compose.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +shopt -s nullglob # Enable nullglob + blocks_dir=docker/blocks docker_dir=docker template_dir=templates @@ -8,6 +10,8 @@ grafana_config_file=conf.tmp grafana_config=config compose_header_file=docker/compose_header.yml +compose_volume_section_file=docker/compose_volume_section.yml +compose_volume_section_create_flag=docker_volume_create_true compose_file=docker-compose.yaml env_file=.env @@ -60,3 +64,27 @@ for dir in $@; do fi done +volume_files=$($blocks_dir/**/$compose_volume_section_create_flag) + +if [[ ${#volume_files[@]} -ne 0 ]]; then + echo "Adding volume section to $compose_file" + cat $compose_volume_section_file >> $compose_file + echo "" >> $compose_file + + for dir in $@; do + current_dir=$blocks_dir/$dir + if [ ! -d "$current_dir" ]; then + echo "$current_dir is not a directory" + exit 1 + fi + + + if [ -f $current_dir/$compose_volume_section_create_flag ]; then + echo "Adding volume for $current_dir to $compose_file" + echo " $dir-data-volume:" >> $compose_file + echo "" >> $compose_file + fi + done + + cat $compose_file +fi diff --git a/devenv/docker/compose_volume_section.yml b/devenv/docker/compose_volume_section.yml new file mode 100644 index 00000000000..f61bd276b2c --- /dev/null +++ b/devenv/docker/compose_volume_section.yml @@ -0,0 +1 @@ +volumes: \ No newline at end of file From 8a099123b97772cf0c22fdc9615792d541d09426 Mon Sep 17 00:00:00 2001 From: Kevin Minehart <5140827+kminehart@users.noreply.github.com> Date: Fri, 19 Sep 2025 17:34:05 -0500 Subject: [PATCH 15/54] CI: npm tag-nightly doesn't need publish (#111424) tag-nightly doesn't need publish --- .github/workflows/release-npm.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release-npm.yml b/.github/workflows/release-npm.yml index 9caeaebe252..e4bed54a7f8 100644 --- a/.github/workflows/release-npm.yml +++ b/.github/workflows/release-npm.yml @@ -135,7 +135,6 @@ jobs: tag-nightly: name: Tag nightly release runs-on: github-hosted-ubuntu-x64-small - needs: publish if: inputs.version_type == 'nightly' steps: @@ -146,4 +145,3 @@ jobs: # TODO: tag the given release with nightly - From a8e900ed9bcf64a1828c7928c2d95798d55dfe24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Sep 2025 15:36:24 +0000 Subject: [PATCH 16/54] Bump github.com/google/wire from 0.6.0 to 0.7.0 (#111281) * Bump github.com/google/wire from 0.6.0 to 0.7.0 Bumps [github.com/google/wire](https://github.com/google/wire) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/google/wire/releases) - [Commits](https://github.com/google/wire/compare/v0.6.0...v0.7.0) --- updated-dependencies: - dependency-name: github.com/google/wire dependency-version: 0.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * update wire --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan McKinley --- apps/iam/go.mod | 2 +- apps/iam/go.sum | 39 ++------------------------------------- go.mod | 2 +- go.sum | 7 ++----- go.work.sum | 3 +-- 5 files changed, 7 insertions(+), 46 deletions(-) diff --git a/apps/iam/go.mod b/apps/iam/go.mod index 7e99af69887..c0fc9ae8bde 100644 --- a/apps/iam/go.mod +++ b/apps/iam/go.mod @@ -200,7 +200,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/google/wire v0.6.0 // indirect + github.com/google/wire v0.7.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.14.2 // indirect github.com/gorilla/mux v1.8.1 // indirect diff --git a/apps/iam/go.sum b/apps/iam/go.sum index 2f6549ac9c1..5b212cbc647 100644 --- a/apps/iam/go.sum +++ b/apps/iam/go.sum @@ -694,12 +694,11 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= -github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -1333,7 +1332,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= @@ -1469,12 +1467,9 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1518,10 +1513,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1578,11 +1569,6 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1621,10 +1607,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1717,24 +1699,15 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053 h1:dHQOQddU4YHS5gY33/6klKjq7Gp3WwMyOXGNp5nzRj8= golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053/go.mod h1:+nZKN+XVh4LCiA9DV3ywrzN4gumyCnKjau3NGb9SGoE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1746,10 +1719,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1818,10 +1787,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk= diff --git a/go.mod b/go.mod index 125078c7571..15102b1c9ca 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/google/go-github/v70 v70.0.0 // indirect; @grafana/grafana-git-ui-sync-team github.com/google/go-querystring v1.1.0 // indirect; @grafana/oss-big-tent github.com/google/uuid v1.6.0 // @grafana/grafana-backend-group - github.com/google/wire v0.6.0 // @grafana/grafana-backend-group + github.com/google/wire v0.7.0 // @grafana/grafana-backend-group github.com/googleapis/gax-go/v2 v2.14.2 // @grafana/grafana-backend-group github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // @grafana/grafana-app-platform-squad diff --git a/go.sum b/go.sum index ace71828555..3fb8164e1ca 100644 --- a/go.sum +++ b/go.sum @@ -1528,7 +1528,6 @@ github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1538,8 +1537,8 @@ github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= -github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= @@ -2785,7 +2784,6 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= @@ -3204,7 +3202,6 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= diff --git a/go.work.sum b/go.work.sum index 6dd02118ee9..918fc645382 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1069,8 +1069,6 @@ github.com/grafana/grafana-app-sdk/logging v0.43.1/go.mod h1:0xrjKSGY5z+NLGuGsXQ github.com/grafana/grafana-app-sdk/logging v0.43.2/go.mod h1:Gh/nBWnspK3oDNWtiM5qUF/fardHzOIEez+SPI3JeHA= github.com/grafana/grafana-app-sdk/plugin v0.41.0 h1:ShUvGpAVzM3UxcsfwS6l/lwW4ytDeTbCQXf8w2P8Yp8= github.com/grafana/grafana-app-sdk/plugin v0.41.0/go.mod h1:YIhimVfAqtOp3kdhxOanaSZjypVKh/bYxf9wfFfhDm0= -github.com/grafana/grafana-app-sdk/plugin v0.45.0 h1:rK4FL5h7SqGBDeUdrUjHVGnwgN6w8deQAKx8gJ27Iew= -github.com/grafana/grafana-app-sdk/plugin v0.45.0/go.mod h1:fZ6lWVMWr0EpkmyocxZ7MTTc9x6b2jYM93+gjlMgVD4= github.com/grafana/grafana-aws-sdk v0.38.2 h1:TzQD0OpWsNjtldi5G5TLDlBRk8OyDf+B5ujcoAu4Dp0= github.com/grafana/grafana-aws-sdk v0.38.2/go.mod h1:j3vi+cXYHEFqjhBGrI6/lw1TNM+dl0Y3f0cSnDOPy+s= github.com/grafana/grafana-aws-sdk v1.0.2 h1:98eBuHYFmgvH0xO9kKf4RBsEsgQRp8EOA/9yhDIpkss= @@ -2035,6 +2033,7 @@ golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.24.1/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= From 04bc71fa6d934d641cc07239b6747253c62a3b51 Mon Sep 17 00:00:00 2001 From: Santiago Date: Mon, 22 Sep 2025 12:29:39 +0200 Subject: [PATCH 17/54] Alert State History: Skip invalid entries when merging streams (#111387) --- pkg/services/ngalert/state/historian/loki.go | 18 ++++-- .../ngalert/state/historian/loki_test.go | 60 ++++++++++++++++++- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/pkg/services/ngalert/state/historian/loki.go b/pkg/services/ngalert/state/historian/loki.go index f31b993ad9d..d5aa7997827 100644 --- a/pkg/services/ngalert/state/historian/loki.go +++ b/pkg/services/ngalert/state/historian/loki.go @@ -175,11 +175,11 @@ func (h *RemoteLokiBackend) Query(ctx context.Context, query models.HistoryQuery } res = append(res, r.Data.Result...) } - return merge(res, uids) + return h.merge(res, uids) } // merge will put all the results in one array sorted by timestamp. -func merge(res []lokiclient.Stream, folderUIDToFilter []string) (*data.Frame, error) { +func (h RemoteLokiBackend) merge(res []lokiclient.Stream, folderUIDToFilter []string) (*data.Frame, error) { filterByFolderUIDMap := make(map[string]struct{}, len(folderUIDToFilter)) for _, uid := range folderUIDToFilter { filterByFolderUIDMap[uid] = struct{}{} @@ -244,19 +244,25 @@ func merge(res []lokiclient.Stream, folderUIDToFilter []string) (*data.Frame, er var entry LokiEntry err := json.Unmarshal([]byte(minEl.V), &entry) if err != nil { - return nil, fmt.Errorf("failed to unmarshal entry: %w", err) + h.log.Warn("failed to unmarshal entry, continuing", "err", err, "entry", minEl.V) + pointers[minElStreamIdx]++ + continue } // Append the minimum element to the merged slice and move the pointer. tsNano := minEl.T.UnixNano() - // TODO: In general, perhaps we should omit the offending line and log, rather than failing the request entirely. streamLbls := res[minElStreamIdx].Stream lblsJson, err := json.Marshal(streamLbls) if err != nil { - return nil, fmt.Errorf("failed to serialize stream labels: %w", err) + // This should in theory never happen, as we're marshalling a map[string]string. + h.log.Warn("failed to serialize stream labels, continuing", "err", err, "labels", streamLbls) + pointers[minElStreamIdx]++ + continue } line, err := jsonifyRow(minEl.V) if err != nil { - return nil, fmt.Errorf("a line was in an invalid format: %w", err) + h.log.Warn("a line was in an invalid format, continuing", "err", err, "line", minEl.V) + pointers[minElStreamIdx]++ + continue } times = append(times, time.Unix(0, tsNano)) diff --git a/pkg/services/ngalert/state/historian/loki_test.go b/pkg/services/ngalert/state/historian/loki_test.go index 99d9aff3b79..f404402c9fe 100644 --- a/pkg/services/ngalert/state/historian/loki_test.go +++ b/pkg/services/ngalert/state/historian/loki_test.go @@ -538,6 +538,62 @@ func TestMerge(t *testing.T) { }), ), }, + { + name: "Should handle bad values", + res: lokiclient.QueryRes{ + Data: lokiclient.QueryData{ + Result: []lokiclient.Stream{ + { + Stream: map[string]string{ + "from": "state-history", + "orgID": "1", + "group": "test-group-1", + "folderUID": "test-folder-1", + }, + Values: []lokiclient.Sample{ + {T: time.Unix(1, 0), V: `{"schemaVersion": 1, "previous": "normal", "current": "pending", "values":{"a": 1.5}, "ruleUID": "test-rule-1"}`}, + {T: time.Unix(5, 0), V: `{"schemaVersion": 1, "previous": "pending", "current": "normal", "values":{"a": 0.5}, "ruleUID": "test-rule-2", "bad_label": "\e"}`}, + }, + }, + { + Stream: map[string]string{ + "from": "state-history", + "orgID": "1", + "group": "test-group-2", + "folderUID": "test-folder-1", + }, + Values: []lokiclient.Sample{ + {T: time.Unix(2, 0), V: `{"schemaVersion": 1, "previous": "pending", "current": "firing", "values":{"a": 2.5}, "ruleUID": "test-rule-3"}`}, + }, + }, + }, + }, + }, + expected: data.NewFrame("states", + data.NewField(dfTime, data.Labels{}, []time.Time{ + time.Unix(1, 0), + time.Unix(2, 0), + }), + data.NewField(dfLine, data.Labels{}, []json.RawMessage{ + toJson(LokiEntry{RuleUID: "test-rule-1", SchemaVersion: 1, Previous: "normal", Current: "pending", Values: jsonifyValues(map[string]float64{"a": 1.5})}), + toJson(LokiEntry{RuleUID: "test-rule-3", SchemaVersion: 1, Previous: "pending", Current: "firing", Values: jsonifyValues(map[string]float64{"a": 2.5})}), + }), + data.NewField(dfLabels, data.Labels{}, []json.RawMessage{ + toJson(map[string]string{ + StateHistoryLabelKey: "state-history", + OrgIDLabel: "1", + GroupLabel: "test-group-1", + FolderUIDLabel: "test-folder-1", + }), + toJson(map[string]string{ + StateHistoryLabelKey: "state-history", + OrgIDLabel: "1", + GroupLabel: "test-group-2", + FolderUIDLabel: "test-folder-1", + }), + }), + ), + }, { name: "should filter streams by folder UID", folderUIDs: []string{"test-folder-1"}, @@ -654,9 +710,11 @@ func TestMerge(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + req := instrumenttest.NewFakeRequester() + loki := createTestLokiBackend(t, req, metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)) expectedJson, err := tc.expected.MarshalJSON() require.NoError(t, err) - m, err := merge(tc.res.Data.Result, tc.folderUIDs) + m, err := loki.merge(tc.res.Data.Result, tc.folderUIDs) require.NoError(t, err) actualJson, err := m.MarshalJSON() assert.NoError(t, err) From bd550d2f063673dc14e72d2785d9281c505ba9b7 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Mon, 22 Sep 2025 08:54:50 -0600 Subject: [PATCH 18/54] Provisioning: Wire up prometheus (#111444) --- apps/provisioning/go.mod | 2 +- apps/provisioning/pkg/controller/status.go | 9 ++-- .../pkg/controller/status_test.go | 5 ++- pkg/operators/provisioning/config.go | 7 ++-- pkg/operators/provisioning/jobs_operator.go | 22 ++++++---- pkg/operators/provisioning/repo_operator.go | 14 ++++--- .../apis/provisioning/controller/health.go | 5 ++- .../provisioning/controller/health_test.go | 15 +++---- .../provisioning/controller/repository.go | 5 +++ .../provisioning/jobs/concurrent_driver.go | 4 ++ .../apis/provisioning/jobs/delete/worker.go | 5 ++- .../provisioning/jobs/delete/worker_test.go | 41 ++++++++++--------- .../apis/provisioning/jobs/export/worker.go | 4 ++ .../provisioning/jobs/export/worker_test.go | 35 ++++++++-------- .../apis/provisioning/jobs/move/worker.go | 5 ++- .../provisioning/jobs/move/worker_test.go | 39 +++++++++--------- .../apis/provisioning/jobs/persistentstore.go | 12 ++++-- .../apis/provisioning/jobs/sync/worker.go | 6 +++ .../provisioning/jobs/sync/worker_test.go | 6 ++- pkg/registry/apis/provisioning/register.go | 18 +++++--- .../apis/provisioning/webhooks/register.go | 7 +++- .../apis/provisioning/webhooks/webhook.go | 4 ++ pkg/server/wire_gen.go | 4 +- 23 files changed, 169 insertions(+), 105 deletions(-) diff --git a/apps/provisioning/go.mod b/apps/provisioning/go.mod index 1ffeb70c4f3..732f69c3671 100644 --- a/apps/provisioning/go.mod +++ b/apps/provisioning/go.mod @@ -11,6 +11,7 @@ require ( github.com/grafana/grafana/pkg/apimachinery v0.0.0-20250804150913-990f1c69ecc2 github.com/grafana/nanogit v0.0.0-20250723104447-68f58f5ecec0 github.com/migueleliasweb/go-github-mock v1.1.0 + github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 golang.org/x/oauth2 v0.30.0 k8s.io/apimachinery v0.34.1 @@ -53,7 +54,6 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect diff --git a/apps/provisioning/pkg/controller/status.go b/apps/provisioning/pkg/controller/status.go index 40ed29624c1..ad63a169415 100644 --- a/apps/provisioning/pkg/controller/status.go +++ b/apps/provisioning/pkg/controller/status.go @@ -7,17 +7,20 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) type RepositoryStatusPatcher struct { - client client.ProvisioningV0alpha1Interface + client client.ProvisioningV0alpha1Interface + registry prometheus.Registerer } -func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *RepositoryStatusPatcher { +func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface, registry prometheus.Registerer) *RepositoryStatusPatcher { return &RepositoryStatusPatcher{ - client: client, + client: client, + registry: registry, } } diff --git a/apps/provisioning/pkg/controller/status_test.go b/apps/provisioning/pkg/controller/status_test.go index 77e7f737ea5..12c21fb8c5b 100644 --- a/apps/provisioning/pkg/controller/status_test.go +++ b/apps/provisioning/pkg/controller/status_test.go @@ -8,6 +8,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1/fake" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -16,7 +17,7 @@ import ( func TestNewRepositoryStatusPatcher(t *testing.T) { client := &fake.FakeProvisioningV0alpha1{} - patcher := NewRepositoryStatusPatcher(client) + patcher := NewRepositoryStatusPatcher(client, prometheus.DefaultRegisterer) require.NotNil(t, patcher) require.Equal(t, client, patcher.client) } @@ -103,7 +104,7 @@ func TestRepositoryStatusPatcher_Patch(t *testing.T) { client.AddReactor("patch", "repositories", tt.reactorFunc) } - patcher := NewRepositoryStatusPatcher(&client) + patcher := NewRepositoryStatusPatcher(&client, prometheus.DefaultRegisterer) err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations...) if tt.expectedError != "" { diff --git a/pkg/operators/provisioning/config.go b/pkg/operators/provisioning/config.go index 98d1fb78d5e..273070ac156 100644 --- a/pkg/operators/provisioning/config.go +++ b/pkg/operators/provisioning/config.go @@ -69,7 +69,7 @@ type provisioningControllerConfig struct { // local_permitted_prefixes = // [provisioning] // repository_types = -func setupFromConfig(cfg *setting.Cfg) (controllerCfg *provisioningControllerConfig, err error) { +func setupFromConfig(cfg *setting.Cfg, registry prometheus.Registerer) (controllerCfg *provisioningControllerConfig, err error) { if cfg == nil { return nil, fmt.Errorf("no configuration available") } @@ -130,7 +130,7 @@ func setupFromConfig(cfg *setting.Cfg) (controllerCfg *provisioningControllerCon return nil, fmt.Errorf("failed to setup decrypter: %w", err) } - repoFactory, err := setupRepoFactory(cfg, decrypter, provisioningClient) + repoFactory, err := setupRepoFactory(cfg, decrypter, provisioningClient, registry) if err != nil { return nil, fmt.Errorf("failed to setup repository getter: %w", err) } @@ -220,6 +220,7 @@ func setupRepoFactory( cfg *setting.Cfg, decrypter repository.Decrypter, provisioningClient *client.Clientset, + registry prometheus.Registerer, ) (repository.Factory, error) { operatorSec := cfg.SectionWithEnvOverrides("operator") provisioningSec := cfg.SectionWithEnvOverrides("provisioning") @@ -246,7 +247,7 @@ func setupRepoFactory( var webhook *webhooks.WebhookExtraBuilder provisioningAppURL := operatorSec.Key("provisioning_server_public_url").String() if provisioningAppURL != "" { - webhook = webhooks.ProvideWebhooks(provisioningAppURL) + webhook = webhooks.ProvideWebhooks(provisioningAppURL, registry) } extras = append(extras, github.Extra( diff --git a/pkg/operators/provisioning/jobs_operator.go b/pkg/operators/provisioning/jobs_operator.go index 1da6600c724..49fa2676717 100644 --- a/pkg/operators/provisioning/jobs_operator.go +++ b/pkg/operators/provisioning/jobs_operator.go @@ -10,6 +10,7 @@ import ( "time" "github.com/grafana/grafana-app-sdk/logging" + "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/tools/cache" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" @@ -33,7 +34,7 @@ func RunJobController(deps server.OperatorDependencies) error { })).With("logger", "provisioning-job-controller") logger.Info("Starting provisioning job controller") - controllerCfg, err := setupJobsControllerFromConfig(deps.Config) + controllerCfg, err := setupJobsControllerFromConfig(deps.Config, deps.Registerer) if err != nil { return fmt.Errorf("failed to setup operator: %w", err) } @@ -94,12 +95,12 @@ func RunJobController(deps server.OperatorDependencies) error { // } jobHistoryWriter := jobs.NewAPIClientHistoryWriter(controllerCfg.provisioningClient.ProvisioningV0alpha1()) - jobStore, err := jobs.NewJobStore(controllerCfg.provisioningClient.ProvisioningV0alpha1(), 30*time.Second) + jobStore, err := jobs.NewJobStore(controllerCfg.provisioningClient.ProvisioningV0alpha1(), 30*time.Second, deps.Registerer) if err != nil { return fmt.Errorf("create API client job store: %w", err) } - workers, err := setupWorkers(controllerCfg) + workers, err := setupWorkers(controllerCfg, deps.Registerer) if err != nil { return fmt.Errorf("setup workers: %w", err) } @@ -120,6 +121,7 @@ func RunJobController(deps server.OperatorDependencies) error { repoGetter, jobHistoryWriter, jobController.InsertNotifications(), + deps.Registerer, workers..., ) if err != nil { @@ -151,8 +153,8 @@ type jobsControllerConfig struct { historyExpiration time.Duration } -func setupJobsControllerFromConfig(cfg *setting.Cfg) (*jobsControllerConfig, error) { - controllerCfg, err := setupFromConfig(cfg) +func setupJobsControllerFromConfig(cfg *setting.Cfg, registry prometheus.Registerer) (*jobsControllerConfig, error) { + controllerCfg, err := setupFromConfig(cfg, registry) if err != nil { return nil, err } @@ -163,12 +165,12 @@ func setupJobsControllerFromConfig(cfg *setting.Cfg) (*jobsControllerConfig, err }, nil } -func setupWorkers(controllerCfg *jobsControllerConfig) ([]jobs.Worker, error) { +func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Registerer) ([]jobs.Worker, error) { clients := controllerCfg.clients parsers := resources.NewParserFactory(clients) resourceLister := resources.NewResourceLister(controllerCfg.unified) repositoryResources := resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister) - statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1()) + statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), registry) workers := make([]jobs.Worker, 0) @@ -180,6 +182,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig) ([]jobs.Worker, error) { nil, // HACK: we have updated the worker to check for nil statusPatcher.Patch, syncer, + registry, ) workers = append(workers, syncWorker) @@ -190,6 +193,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig) ([]jobs.Worker, error) { repositoryResources, export.ExportAll, stageIfPossible, + registry, ) workers = append(workers, exportWorker) @@ -204,11 +208,11 @@ func setupWorkers(controllerCfg *jobsControllerConfig) ([]jobs.Worker, error) { workers = append(workers, migrationWorker) // Delete - deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, repositoryResources) + deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, repositoryResources, registry) workers = append(workers, deleteWorker) // Move - moveWorker := move.NewWorker(syncWorker, stageIfPossible, repositoryResources) + moveWorker := move.NewWorker(syncWorker, stageIfPossible, repositoryResources, registry) workers = append(workers, moveWorker) return workers, nil diff --git a/pkg/operators/provisioning/repo_operator.go b/pkg/operators/provisioning/repo_operator.go index 6162007f8e4..b64cb991777 100644 --- a/pkg/operators/provisioning/repo_operator.go +++ b/pkg/operators/provisioning/repo_operator.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana-app-sdk/logging" appcontroller "github.com/grafana/grafana/apps/provisioning/pkg/controller" + "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/tools/cache" "github.com/grafana/grafana/pkg/registry/apis/provisioning/controller" @@ -28,7 +29,7 @@ func RunRepoController(deps server.OperatorDependencies) error { })).With("logger", "provisioning-repo-controller") logger.Info("Starting provisioning repo controller") - controllerCfg, err := getRepoControllerConfig(deps.Config) + controllerCfg, err := getRepoControllerConfig(deps.Config, deps.Registerer) if err != nil { return fmt.Errorf("failed to setup operator: %w", err) } @@ -50,12 +51,12 @@ func RunRepoController(deps server.OperatorDependencies) error { ) resourceLister := resources.NewResourceLister(controllerCfg.unified) - jobs, err := jobs.NewJobStore(controllerCfg.provisioningClient.ProvisioningV0alpha1(), 30*time.Second) + jobs, err := jobs.NewJobStore(controllerCfg.provisioningClient.ProvisioningV0alpha1(), 30*time.Second, deps.Registerer) if err != nil { return fmt.Errorf("create API client job store: %w", err) } - statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1()) - healthChecker := controller.NewHealthChecker(statusPatcher) + statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), deps.Registerer) + healthChecker := controller.NewHealthChecker(statusPatcher, deps.Registerer) repoInformer := informerFactory.Provisioning().V0alpha1().Repositories() controller, err := controller.NewRepositoryController( @@ -68,6 +69,7 @@ func RunRepoController(deps server.OperatorDependencies) error { nil, // dualwrite -- standalone operator assumes it is backed by unified storage healthChecker, statusPatcher, + deps.Registerer, ) if err != nil { return fmt.Errorf("failed to create repository controller: %w", err) @@ -87,8 +89,8 @@ type repoControllerConfig struct { workerCount int } -func getRepoControllerConfig(cfg *setting.Cfg) (*repoControllerConfig, error) { - controllerCfg, err := setupFromConfig(cfg) +func getRepoControllerConfig(cfg *setting.Cfg, registry prometheus.Registerer) (*repoControllerConfig, error) { + controllerCfg, err := setupFromConfig(cfg, registry) if err != nil { return nil, err } diff --git a/pkg/registry/apis/provisioning/controller/health.go b/pkg/registry/apis/provisioning/controller/health.go index f6bad8fff6b..0d24e457274 100644 --- a/pkg/registry/apis/provisioning/controller/health.go +++ b/pkg/registry/apis/provisioning/controller/health.go @@ -7,6 +7,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/prometheus/client_golang/prometheus" ) // StatusPatcher defines the interface for updating repository status @@ -19,12 +20,14 @@ type StatusPatcher interface { // HealthChecker provides unified health checking for repositories type HealthChecker struct { statusPatcher StatusPatcher + registry prometheus.Registerer } // NewHealthChecker creates a new health checker -func NewHealthChecker(statusPatcher StatusPatcher) *HealthChecker { +func NewHealthChecker(statusPatcher StatusPatcher, registry prometheus.Registerer) *HealthChecker { return &HealthChecker{ statusPatcher: statusPatcher, + registry: registry, } } diff --git a/pkg/registry/apis/provisioning/controller/health_test.go b/pkg/registry/apis/provisioning/controller/health_test.go index a42cd35c7aa..c9081875b2a 100644 --- a/pkg/registry/apis/provisioning/controller/health_test.go +++ b/pkg/registry/apis/provisioning/controller/health_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,7 +19,7 @@ import ( func TestNewHealthChecker(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) assert.NotNil(t, hc) assert.Equal(t, mockPatcher, hc.statusPatcher) @@ -135,7 +136,7 @@ func TestShouldCheckHealth(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) result := hc.ShouldCheckHealth(tt.repo) assert.Equal(t, tt.expected, result) @@ -222,7 +223,7 @@ func TestHasRecentFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) result := hc.HasRecentFailure(tt.healthStatus, tt.failureType) assert.Equal(t, tt.expected, result) @@ -264,7 +265,7 @@ func TestRecordFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) repo := &provisioning.Repository{ Status: provisioning.RepositoryStatus{ @@ -309,7 +310,7 @@ func TestRecordFailure(t *testing.T) { func TestRecordFailureFunction(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) testErr := errors.New("test error") result := hc.recordFailure(provisioning.HealthFailureHook, testErr) @@ -446,7 +447,7 @@ func TestRefreshHealth(t *testing.T) { testError: tt.testError, } - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) if tt.expectPatch { if tt.patchError != nil { @@ -556,7 +557,7 @@ func TestHasHealthStatusChanged(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher) + hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) result := hc.hasHealthStatusChanged(tt.old, tt.new) assert.Equal(t, tt.expected, result) diff --git a/pkg/registry/apis/provisioning/controller/repository.go b/pkg/registry/apis/provisioning/controller/repository.go index 1b59c5ac2ef..9ddaf4fa476 100644 --- a/pkg/registry/apis/provisioning/controller/repository.go +++ b/pkg/registry/apis/provisioning/controller/repository.go @@ -25,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "github.com/prometheus/client_golang/prometheus" ) const loggerName = "provisioning-repository-controller" @@ -59,6 +60,8 @@ type RepositoryController struct { keyFunc func(obj any) (string, error) queue workqueue.TypedRateLimitingInterface[*queueItem] + + registry prometheus.Registerer } // NewRepositoryController creates new RepositoryController. @@ -72,6 +75,7 @@ func NewRepositoryController( dualwrite dualwrite.Service, healthChecker *HealthChecker, statusPatcher StatusPatcher, + registry prometheus.Registerer, ) (*RepositoryController, error) { rc := &RepositoryController{ client: provisioningClient, @@ -93,6 +97,7 @@ func NewRepositoryController( jobs: jobs, logger: logging.DefaultLogger.With("logger", loggerName), dualwrite: dualwrite, + registry: registry, } _, err := repoInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/pkg/registry/apis/provisioning/jobs/concurrent_driver.go b/pkg/registry/apis/provisioning/jobs/concurrent_driver.go index 0504fc414d5..4da84e4ec84 100644 --- a/pkg/registry/apis/provisioning/jobs/concurrent_driver.go +++ b/pkg/registry/apis/provisioning/jobs/concurrent_driver.go @@ -7,6 +7,7 @@ import ( "time" "github.com/grafana/grafana-app-sdk/logging" + "github.com/prometheus/client_golang/prometheus" ) // ConcurrentJobDriver manages multiple jobDriver instances for concurrent job processing. @@ -20,6 +21,7 @@ type ConcurrentJobDriver struct { repoGetter RepoGetter historicJobs HistoryWriter workers []Worker + registry prometheus.Registerer notifications chan struct{} } @@ -31,6 +33,7 @@ func NewConcurrentJobDriver( repoGetter RepoGetter, historicJobs HistoryWriter, notifications chan struct{}, + registry prometheus.Registerer, workers ...Worker, ) (*ConcurrentJobDriver, error) { if numDrivers <= 0 { @@ -66,6 +69,7 @@ func NewConcurrentJobDriver( historicJobs: historicJobs, workers: workers, notifications: notifications, + registry: registry, }, nil } diff --git a/pkg/registry/apis/provisioning/jobs/delete/worker.go b/pkg/registry/apis/provisioning/jobs/delete/worker.go index 11d78e73502..738876d1517 100644 --- a/pkg/registry/apis/provisioning/jobs/delete/worker.go +++ b/pkg/registry/apis/provisioning/jobs/delete/worker.go @@ -12,19 +12,22 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/prometheus/client_golang/prometheus" ) type Worker struct { syncWorker jobs.Worker wrapFn repository.WrapWithStageFn resourcesFactory resources.RepositoryResourcesFactory + registry prometheus.Registerer } -func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory) *Worker { +func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory, registry prometheus.Registerer) *Worker { return &Worker{ syncWorker: syncWorker, wrapFn: wrapFn, resourcesFactory: resourcesFactory, + registry: registry, } } diff --git a/pkg/registry/apis/provisioning/jobs/delete/worker_test.go b/pkg/registry/apis/provisioning/jobs/delete/worker_test.go index 1bcfed8d4cc..c5fd53282f8 100644 --- a/pkg/registry/apis/provisioning/jobs/delete/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/delete/worker_test.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -73,7 +74,7 @@ func TestDeleteWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -87,7 +88,7 @@ func TestDeleteWorker_ProcessMissingDeleteSettings(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing delete settings") } @@ -116,7 +117,7 @@ func TestDeleteWorker_ProcessNotReaderWriter(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: delete job submitted targeting repository that is not a ReaderWriter") } @@ -139,7 +140,7 @@ func TestDeleteWorker_ProcessWrapFnError(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: stage failed") } @@ -187,7 +188,7 @@ func TestDeleteWorker_ProcessDeleteFilesSuccess(t *testing.T) { return result.Path == "test/path2" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -225,7 +226,7 @@ func TestDeleteWorker_ProcessDeleteFilesWithError(t *testing.T) { })).Return() mockProgress.On("TooManyErrors").Return(errors.New("too many errors")) - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: too many errors") } @@ -269,7 +270,7 @@ func TestDeleteWorker_ProcessWithSyncWorker(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -309,7 +310,7 @@ func TestDeleteWorker_ProcessSyncWorkerError(t *testing.T) { syncError := errors.New("sync failed") mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.Anything, mockProgress).Return(syncError) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "pull resources: sync failed") } @@ -378,7 +379,7 @@ func TestDeleteWorker_deleteFiles(t *testing.T) { } } - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) err := worker.deleteFiles(context.Background(), mockRepo, mockProgress, opts, tt.paths...) if tt.expectedError != "" { @@ -473,7 +474,7 @@ func TestDeleteWorker_ProcessWithResourceRefs(t *testing.T) { return result.Path == "folders/test-folder.json" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) @@ -531,7 +532,7 @@ func TestDeleteWorker_ProcessResourceRefsOnly(t *testing.T) { return result.Path == "dashboards/test-dashboard.json" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -596,7 +597,7 @@ func TestDeleteWorker_ProcessResourceResolutionError(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) // Should succeed even with resource resolution error } @@ -635,7 +636,7 @@ func TestDeleteWorker_ProcessResourcesFactoryError(t *testing.T) { mockProgress.On("StrictMaxErrors", 1).Return() mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: create repository resources client: failed to create repository resources client") } @@ -671,7 +672,7 @@ func TestDeleteWorker_ProcessResourceRefsNotReaderWriter(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: delete job submitted targeting repository that is not a ReaderWriter") } @@ -724,7 +725,7 @@ func TestDeleteWorker_ProcessResourceResolutionTooManyErrors(t *testing.T) { })).Return() mockProgress.On("TooManyErrors").Return(errors.New("too many errors")) - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: too many errors") } @@ -821,7 +822,7 @@ func TestDeleteWorker_ProcessMixedResourcesWithPartialFailure(t *testing.T) { return result.Path == "folders/valid-folder.json" && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) // Should succeed overall, with only the failed resource recorded as error } @@ -919,7 +920,7 @@ func TestDeleteWorker_ProcessWithPathDeduplication(t *testing.T) { return result.Path == "dashboards/unique-dashboard.json" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) @@ -1036,7 +1037,7 @@ func TestDeleteWorker_RefURLsSetWithRef(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -1092,7 +1093,7 @@ func TestDeleteWorker_RefURLsNotSetWithoutRef(t *testing.T) { }, } - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -1142,7 +1143,7 @@ func TestDeleteWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) diff --git a/pkg/registry/apis/provisioning/jobs/export/worker.go b/pkg/registry/apis/provisioning/jobs/export/worker.go index e7f5404d6cc..f7def54ac1f 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/prometheus/client_golang/prometheus" ) //go:generate mockery --name ExportFn --structname MockExportFn --inpackage --filename mock_export_fn.go --with-expecter @@ -23,6 +24,7 @@ type ExportWorker struct { repositoryResources resources.RepositoryResourcesFactory exportFn ExportFn wrapWithStageFn WrapWithStageFn + registry prometheus.Registerer } func NewExportWorker( @@ -30,12 +32,14 @@ func NewExportWorker( repositoryResources resources.RepositoryResourcesFactory, exportFn ExportFn, wrapWithStageFn WrapWithStageFn, + registry prometheus.Registerer, ) *ExportWorker { return &ExportWorker{ clientFactory: clientFactory, repositoryResources: repositoryResources, exportFn: exportFn, wrapWithStageFn: wrapWithStageFn, + registry: registry, } } diff --git a/pkg/registry/apis/provisioning/jobs/export/worker_test.go b/pkg/registry/apis/provisioning/jobs/export/worker_test.go index 21cb19d0bda..88b69691667 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" mock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -54,7 +55,7 @@ func TestExportWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r := NewExportWorker(nil, nil, nil, nil) + r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) got := r.IsSupported(context.Background(), tt.job) require.Equal(t, tt.want, got) }) @@ -68,7 +69,7 @@ func TestExportWorker_ProcessNoExportSettings(t *testing.T) { }, } - r := NewExportWorker(nil, nil, nil, nil) + r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) err := r.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing export settings") } @@ -91,7 +92,7 @@ func TestExportWorker_ProcessWriteNotAllowed(t *testing.T) { }, }) - r := NewExportWorker(nil, nil, nil, nil) + r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, nil) require.EqualError(t, err, "this repository is read only") } @@ -115,7 +116,7 @@ func TestExportWorker_ProcessBranchNotAllowedForLocal(t *testing.T) { }, }) - r := NewExportWorker(nil, nil, nil, nil) + r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, nil) require.EqualError(t, err, "this repository does not support the branch workflow") } @@ -147,7 +148,7 @@ func TestExportWorker_ProcessFailedToCreateClients(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute) + r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) mockProgress := jobs.NewMockJobProgressRecorder(t) err := r.Process(context.Background(), mockRepo, job, mockProgress) @@ -183,7 +184,7 @@ func TestExportWorker_ProcessNotReaderWriter(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute) + r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export job submitted targeting repository that is not a ReaderWriter") } @@ -219,7 +220,7 @@ func TestExportWorker_ProcessRepositoryResourcesError(t *testing.T) { mockStageFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOpts repository.StageOptions, fn func(repository.Repository, bool) error) error { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, nil, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "create repository resource client: failed to create repository resources client") } @@ -270,7 +271,7 @@ func TestExportWorker_ProcessStageOptions(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -351,7 +352,7 @@ func TestExportWorker_ProcessStageOptionsWithBranch(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) }) @@ -394,7 +395,7 @@ func TestExportWorker_ProcessExportFnError(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export failed") } @@ -422,7 +423,7 @@ func TestExportWorker_ProcessWrapWithStageFnError(t *testing.T) { mockStageFn := NewMockWrapWithStageFn(t) mockStageFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(errors.New("stage failed")) - r := NewExportWorker(nil, nil, nil, mockStageFn.Execute) + r := NewExportWorker(nil, nil, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "stage failed") } @@ -448,7 +449,7 @@ func TestExportWorker_ProcessBranchNotAllowedForStageableRepositories(t *testing mockProgress := jobs.NewMockJobProgressRecorder(t) // No progress messages expected in current implementation - r := NewExportWorker(nil, nil, nil, nil) + r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "this repository does not support the branch workflow") } @@ -499,7 +500,7 @@ func TestExportWorker_ProcessGitRepository(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -545,7 +546,7 @@ func TestExportWorker_ProcessGitRepositoryExportFnError(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export failed") } @@ -608,7 +609,7 @@ func TestExportWorker_RefURLsSetWithBranch(t *testing.T) { return fn(mockReaderWriter, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -664,7 +665,7 @@ func TestExportWorker_RefURLsNotSetWithoutBranch(t *testing.T) { return fn(mockReaderWriter, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -720,7 +721,7 @@ func TestExportWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { return fn(mockReaderWriter, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) diff --git a/pkg/registry/apis/provisioning/jobs/move/worker.go b/pkg/registry/apis/provisioning/jobs/move/worker.go index 790630f101e..35c4e6b0342 100644 --- a/pkg/registry/apis/provisioning/jobs/move/worker.go +++ b/pkg/registry/apis/provisioning/jobs/move/worker.go @@ -14,19 +14,22 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/safepath" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/prometheus/client_golang/prometheus" ) type Worker struct { syncWorker jobs.Worker wrapFn repository.WrapWithStageFn resourcesFactory resources.RepositoryResourcesFactory + registry prometheus.Registerer } -func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory) *Worker { +func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory, registry prometheus.Registerer) *Worker { return &Worker{ syncWorker: syncWorker, wrapFn: wrapFn, resourcesFactory: resourcesFactory, + registry: registry, } } diff --git a/pkg/registry/apis/provisioning/jobs/move/worker_test.go b/pkg/registry/apis/provisioning/jobs/move/worker_test.go index 3f9ea095d60..f8f82f20142 100644 --- a/pkg/registry/apis/provisioning/jobs/move/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/move/worker_test.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/safepath" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -75,7 +76,7 @@ func TestMoveWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -89,7 +90,7 @@ func TestMoveWorker_ProcessMissingMoveSettings(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing move settings") } @@ -104,7 +105,7 @@ func TestMoveWorker_ProcessMissingTargetPath(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "target path is required for move operation") } @@ -120,7 +121,7 @@ func TestMoveWorker_ProcessInvalidTargetPath(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "target path must be a directory (should end with '/')") } @@ -152,7 +153,7 @@ func TestMoveWorker_ProcessNotReaderWriter(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: move job submitted targeting repository that is not a ReaderWriter") } @@ -176,7 +177,7 @@ func TestMoveWorker_ProcessWrapFnError(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: stage failed") } @@ -223,7 +224,7 @@ func TestMoveWorker_ProcessMoveFilesSuccess(t *testing.T) { return result.Path == "test/path2" && result.Action == repository.FileActionRenamed && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -262,7 +263,7 @@ func TestMoveWorker_ProcessMoveFilesWithError(t *testing.T) { })).Return() mockProgress.On("TooManyErrors").Return(errors.New("too many errors")) - worker := NewWorker(nil, mockWrapFn.Execute, nil) + worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: too many errors") } @@ -307,7 +308,7 @@ func TestMoveWorker_ProcessWithSyncWorker(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -348,7 +349,7 @@ func TestMoveWorker_ProcessSyncWorkerError(t *testing.T) { syncError := errors.New("sync failed") mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.Anything, mockProgress).Return(syncError) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "pull resources: sync failed") } @@ -429,7 +430,7 @@ func TestMoveWorker_moveFiles(t *testing.T) { } } - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) err := worker.moveFiles(context.Background(), mockRepo, mockProgress, opts, tt.paths...) if tt.expectedError != "" { @@ -485,7 +486,7 @@ func TestMoveWorker_constructTargetPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewWorker(nil, nil, nil) + worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) result := worker.constructTargetPath(tt.jobTargetPath, tt.sourcePath) require.Equal(t, tt.expectedTarget, result) }) @@ -554,7 +555,7 @@ func TestMoveWorker_ProcessWithResourceReferences(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -615,7 +616,7 @@ func TestMoveWorker_ProcessResourceReferencesError(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) // Should continue despite individual resource errors } @@ -655,7 +656,7 @@ func TestMoveWorker_ProcessResourcesFactoryError(t *testing.T) { factoryError := errors.New("failed to create resources client") mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(nil, factoryError) - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: create repository resources client: failed to create resources client") } @@ -773,7 +774,7 @@ func TestMoveWorker_resolveResourcesToPaths(t *testing.T) { } } - worker := NewWorker(nil, nil, mockResourcesFactory) + worker := NewWorker(nil, nil, mockResourcesFactory, prometheus.DefaultRegisterer) paths, err := worker.resolveResourcesToPaths(context.Background(), mockRepo, mockProgress, tt.resources) if tt.expectedError != "" { @@ -885,7 +886,7 @@ func TestMoveWorker_RefURLsSetWithRef(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -942,7 +943,7 @@ func TestMoveWorker_RefURLsNotSetWithoutRef(t *testing.T) { }, } - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -993,7 +994,7 @@ func TestMoveWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) diff --git a/pkg/registry/apis/provisioning/jobs/persistentstore.go b/pkg/registry/apis/provisioning/jobs/persistentstore.go index 5604e7bff0c..cd5b5c1561b 100644 --- a/pkg/registry/apis/provisioning/jobs/persistentstore.go +++ b/pkg/registry/apis/provisioning/jobs/persistentstore.go @@ -18,6 +18,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/prometheus/client_golang/prometheus" ) const ( @@ -72,18 +73,21 @@ type persistentStore struct { // expiry is the time after which a job is considered abandoned. // If a job is abandoned, it will have its claim cleaned up periodically. expiry time.Duration + + registry prometheus.Registerer } // NewJobStore creates a new job queue implementation using the API client. -func NewJobStore(provisioningClient client.ProvisioningV0alpha1Interface, expiry time.Duration) (*persistentStore, error) { +func NewJobStore(provisioningClient client.ProvisioningV0alpha1Interface, expiry time.Duration, registry prometheus.Registerer) (*persistentStore, error) { if expiry <= 0 { expiry = time.Second * 30 } return &persistentStore{ - client: provisioningClient, - clock: time.Now, - expiry: expiry, + client: provisioningClient, + clock: time.Now, + expiry: expiry, + registry: registry, }, nil } diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker.go b/pkg/registry/apis/provisioning/jobs/sync/worker.go index 87ef447ff7f..a2e7f7feb69 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "github.com/prometheus/client_golang/prometheus" ) //go:generate mockery --name RepositoryPatchFn --structname MockRepositoryPatchFn --inpackage --filename repository_patch_fn_mock.go --with-expecter @@ -32,6 +33,9 @@ type SyncWorker struct { // Sync functions syncer Syncer + + // Registry for metrics + registry prometheus.Registerer } func NewSyncWorker( @@ -40,6 +44,7 @@ func NewSyncWorker( storageStatus dualwrite.Service, patchStatus RepositoryPatchFn, syncer Syncer, + registry prometheus.Registerer, ) *SyncWorker { return &SyncWorker{ clients: clients, @@ -47,6 +52,7 @@ func NewSyncWorker( patchStatus: patchStatus, storageStatus: storageStatus, syncer: syncer, + registry: registry, } } diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go index 901c577aab8..cce78138148 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,7 +44,7 @@ func TestSyncWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewSyncWorker(nil, nil, nil, nil, nil) + worker := NewSyncWorker(nil, nil, nil, nil, nil, prometheus.DefaultRegisterer) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -62,7 +63,7 @@ func TestSyncWorker_ProcessNotReaderWriter(t *testing.T) { }) fakeDualwrite := dualwrite.NewMockService(t) fakeDualwrite.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() - worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil) + worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil, prometheus.DefaultRegisterer) err := worker.Process(context.Background(), repo, provisioning.Job{}, jobs.NewMockJobProgressRecorder(t)) require.EqualError(t, err, "sync job submitted for repository that does not support read-write -- this is a bug") } @@ -530,6 +531,7 @@ func TestSyncWorker_Process(t *testing.T) { dualwriteService, repositoryPatchFn.Execute, syncer, + prometheus.DefaultRegisterer, ) // Create test job diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index c0fdc90ca34..08a5c5cc034 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -117,6 +117,7 @@ type APIBuilder struct { extraWorkers []jobs.Worker restConfigGetter func(context.Context) (*clientrest.Config, error) + registry prometheus.Registerer } // NewAPIBuilder creates an API builder. @@ -139,6 +140,7 @@ func NewAPIBuilder( allowedTargets []provisioning.SyncTargetType, restConfigGetter func(context.Context) (*clientrest.Config, error), allowImageRendering bool, + registry prometheus.Registerer, newStandaloneClientFactoryFunc func(loopbackConfigProvider apiserver.RestConfigProvider) resources.ClientFactory, // optional, only used for standalone apiserver ) *APIBuilder { var clients resources.ClientFactory @@ -169,6 +171,7 @@ func NewAPIBuilder( allowedTargets: allowedTargets, restConfigGetter: restConfigGetter, allowImageRendering: allowImageRendering, + registry: registry, } for _, builder := range extraBuilders { @@ -254,6 +257,7 @@ func RegisterAPIService( allowedTargets, nil, // will use loopback instead cfg.ProvisioningAllowImageRendering, + reg, nil, ) apiregistration.RegisterAPI(builder) @@ -707,13 +711,13 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.client = c.ProvisioningV0alpha1() // Initialize the API client-based job store - b.jobs, err = jobs.NewJobStore(b.client, 30*time.Second) + b.jobs, err = jobs.NewJobStore(b.client, 30*time.Second, b.registry) if err != nil { return fmt.Errorf("create API client job store: %w", err) } - b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient()) - b.healthChecker = controller.NewHealthChecker(b.statusPatcher) + b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient(), b.registry) + b.healthChecker = controller.NewHealthChecker(b.statusPatcher, b.registry) // if running solely CRUD, skip the rest of the setup if b.onlyApiServer { @@ -739,6 +743,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.repositoryResources, export.ExportAll, stageIfPossible, + b.registry, ) syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync) @@ -748,6 +753,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.storageStatus, b.statusPatcher.Patch, syncer, + b.registry, ) signerFactory := signature.NewSignerFactory(b.clients) legacyResources := migrate.NewLegacyResourcesMigrator( @@ -779,8 +785,8 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.storageStatus, ) - deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources) - moveWorker := movepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources) + deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources, b.registry) + moveWorker := movepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources, b.registry) workers := []jobs.Worker{ deleteWorker, exportWorker, @@ -815,6 +821,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH 30*time.Second, // Lease renewal interval b.jobs, repoGetter, jobHistoryWriter, jobController.InsertNotifications(), + b.registry, workers..., ) if err != nil { @@ -837,6 +844,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.storageStatus, b.GetHealthChecker(), b.statusPatcher, + b.registry, ) if err != nil { return err diff --git a/pkg/registry/apis/provisioning/webhooks/register.go b/pkg/registry/apis/provisioning/webhooks/register.go index ce357188bad..6dc55a5b0de 100644 --- a/pkg/registry/apis/provisioning/webhooks/register.go +++ b/pkg/registry/apis/provisioning/webhooks/register.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/prometheus/client_golang/prometheus" ) // WebhookExtraBuilder is a function that returns an ExtraBuilder. @@ -63,6 +64,7 @@ func ProvideWebhooksWithImages( renderer rendering.Service, blobstore resource.ResourceClient, configProvider apiserver.RestConfigProvider, + registry prometheus.Registerer, ) *WebhookExtraBuilder { urlProvider := func(_ string) string { return cfg.AppURL @@ -82,6 +84,7 @@ func ProvideWebhooksWithImages( isPublic, b, screenshotRenderer, + registry, ) evaluator := pullrequest.NewEvaluator(screenshotRenderer, parsers, urlProvider) @@ -98,7 +101,7 @@ func ProvideWebhooksWithImages( } } -func ProvideWebhooks(provisioningURL string) *WebhookExtraBuilder { +func ProvideWebhooks(provisioningURL string, registry prometheus.Registerer) *WebhookExtraBuilder { urlProvider := func(_ string) string { return provisioningURL } @@ -110,7 +113,7 @@ func ProvideWebhooks(provisioningURL string) *WebhookExtraBuilder { urlProvider: urlProvider, ExtraBuilder: func(b *provisioningapis.APIBuilder) provisioningapis.Extra { screenshotRenderer := pullrequest.NewNoOpRenderer() - webhook := NewWebhookConnector(isPublic, b, screenshotRenderer) + webhook := NewWebhookConnector(isPublic, b, screenshotRenderer, registry) return NewWebhookExtra(webhook) }, diff --git a/pkg/registry/apis/provisioning/webhooks/webhook.go b/pkg/registry/apis/provisioning/webhooks/webhook.go index 0997818d692..321f08fdafb 100644 --- a/pkg/registry/apis/provisioning/webhooks/webhook.go +++ b/pkg/registry/apis/provisioning/webhooks/webhook.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/identity" provisioningapis "github.com/grafana/grafana/pkg/registry/apis/provisioning" "github.com/grafana/grafana/pkg/registry/apis/provisioning/webhooks/pullrequest" + "github.com/prometheus/client_golang/prometheus" ) type WebhookRepository interface { @@ -34,6 +35,7 @@ type webhookConnector struct { webhooksEnabled bool core *provisioningapis.APIBuilder renderer pullrequest.ScreenshotRenderer + registry prometheus.Registerer } func NewWebhookConnector( @@ -41,11 +43,13 @@ func NewWebhookConnector( // TODO: use interface for this core *provisioningapis.APIBuilder, renderer pullrequest.ScreenshotRenderer, + registry prometheus.Registerer, ) *webhookConnector { return &webhookConnector{ webhooksEnabled: webhooksEnabled, core: core, renderer: renderer, + registry: registry, } } diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index f0f9de53f93..eba4ca46baf 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -827,7 +827,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api userStorageAPIBuilder := userstorage.RegisterAPIService(featureToggles, apiserverService, registerer) apiBuilder := preferences.RegisterAPIService(cfg, featureToggles, sqlStore, prefService, starService, userService, apiserverService) legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, dashboardPermissionsService, accessControl, featureToggles) - webhookExtraBuilder := webhooks.ProvideWebhooksWithImages(cfg, renderingService, resourceClient, eventualRestConfigProvider) + webhookExtraBuilder := webhooks.ProvideWebhooksWithImages(cfg, renderingService, resourceClient, eventualRestConfigProvider, registerer) v3 := extras.ProvideProvisioningExtraAPIs(webhookExtraBuilder) pullRequestWorker := pullrequest.ProvidePullRequestWorker(cfg, renderingService, resourceClient, eventualRestConfigProvider) v4 := extras.ProvideExtraWorkers(pullRequestWorker) @@ -1432,7 +1432,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac userStorageAPIBuilder := userstorage.RegisterAPIService(featureToggles, apiserverService, registerer) apiBuilder := preferences.RegisterAPIService(cfg, featureToggles, sqlStore, prefService, starService, userService, apiserverService) legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, dashboardPermissionsService, accessControl, featureToggles) - webhookExtraBuilder := webhooks.ProvideWebhooksWithImages(cfg, renderingService, resourceClient, eventualRestConfigProvider) + webhookExtraBuilder := webhooks.ProvideWebhooksWithImages(cfg, renderingService, resourceClient, eventualRestConfigProvider, registerer) v3 := extras.ProvideProvisioningExtraAPIs(webhookExtraBuilder) pullRequestWorker := pullrequest.ProvidePullRequestWorker(cfg, renderingService, resourceClient, eventualRestConfigProvider) v4 := extras.ProvideExtraWorkers(pullRequestWorker) From 28c19036f1d3632a07c7eef8b1aacf31ab7652e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Mon, 22 Sep 2025 19:55:27 +0200 Subject: [PATCH 19/54] ButtonSelect: Fixes menu shadow (fixes issue with RefreshPicker) (#111431) * ButtonSelect: Fixes menu shadow * Update e2e tests to look in portal --- .../tests/usePluginLinks.spec.ts | 7 +- .../src/components/Dropdown/ButtonSelect.tsx | 104 +++++------------- 2 files changed, 30 insertions(+), 81 deletions(-) diff --git a/e2e-playwright/test-plugins/grafana-extensionstest-app/tests/usePluginLinks.spec.ts b/e2e-playwright/test-plugins/grafana-extensionstest-app/tests/usePluginLinks.spec.ts index 852fe00fa85..11c20cd9702 100644 --- a/e2e-playwright/test-plugins/grafana-extensionstest-app/tests/usePluginLinks.spec.ts +++ b/e2e-playwright/test-plugins/grafana-extensionstest-app/tests/usePluginLinks.spec.ts @@ -3,13 +3,14 @@ import { test, expect } from '@grafana/plugin-e2e'; import pluginJson from '../plugin.json'; import testApp3pluginJson from '../plugins/grafana-extensionexample3-app/plugin.json'; import { testIds } from '../testIds'; +import { selectors } from '@grafana/e2e-selectors'; test.describe('grafana-extensionstest-app', { tag: ['@plugins'] }, () => { test('should extend the actions menu with a link to a-app plugin', async ({ page }) => { await page.goto(`/a/${pluginJson.id}/added-links`); const section = await page.getByTestId(testIds.addedLinksPage.section1); await section.getByTestId(testIds.actions.button).click(); - await page.getByTestId(testIds.container).getByText('Go to A').click(); + await page.getByTestId(selectors.components.Portal.container).getByText('Go to A').click(); await page.getByTestId(testIds.modal.open).click(); await expect(page.getByTestId(testIds.appA.container)).toBeVisible(); }); @@ -18,7 +19,7 @@ test.describe('grafana-extensionstest-app', { tag: ['@plugins'] }, () => { await page.goto(`/a/${pluginJson.id}/added-links`); const section = await page.getByTestId(testIds.addedLinksPage.section1); await section.getByTestId(testIds.actions.button).click(); - await page.getByTestId(testIds.container).getByText('Open from B').click(); + await page.getByTestId(selectors.components.Portal.container).getByText('Open from B').click(); await expect(page.getByTestId(testIds.appB.modal)).toBeVisible(); }); @@ -26,7 +27,7 @@ test.describe('grafana-extensionstest-app', { tag: ['@plugins'] }, () => { await page.goto(`/a/${pluginJson.id}/added-links`); const section = await page.getByTestId(testIds.addedLinksPage.section1); await section.getByTestId(testIds.actions.button).click(); - await page.getByTestId(testIds.container).getByText('Basic link').click(); + await page.getByTestId(selectors.components.Portal.container).getByText('Basic link').click(); await page.getByTestId(testIds.modal.open).click(); await expect(page.getByTestId(testIds.appA.container)).toBeVisible(); }); diff --git a/packages/grafana-ui/src/components/Dropdown/ButtonSelect.tsx b/packages/grafana-ui/src/components/Dropdown/ButtonSelect.tsx index 199aa1c279e..12e3eb08354 100644 --- a/packages/grafana-ui/src/components/Dropdown/ButtonSelect.tsx +++ b/packages/grafana-ui/src/components/Dropdown/ButtonSelect.tsx @@ -1,17 +1,15 @@ -import { css } from '@emotion/css'; -import { autoUpdate, offset, useClick, useDismiss, useFloating, useInteractions } from '@floating-ui/react'; -import { FocusScope } from '@react-aria/focus'; import { memo, HTMLAttributes, useState } from 'react'; -import { GrafanaTheme2, SelectableValue } from '@grafana/data'; +import { SelectableValue } from '@grafana/data'; -import { useStyles2 } from '../../themes/ThemeContext'; -import { getPositioningMiddleware } from '../../utils/floating'; import { Menu } from '../Menu/Menu'; import { MenuItem } from '../Menu/MenuItem'; +import { ScrollContainer } from '../ScrollContainer/ScrollContainer'; import { ToolbarButton, ToolbarButtonVariant } from '../ToolbarButton/ToolbarButton'; import { PopoverContent } from '../Tooltip/types'; +import { Dropdown } from './Dropdown'; + export interface Props extends HTMLAttributes { className?: string; options: Array>; @@ -29,70 +27,34 @@ export interface Props extends HTMLAttributes { */ const ButtonSelectComponent = (props: Props) => { const { className, options, value, onChange, narrow, variant, ...restProps } = props; - const styles = useStyles2(getStyles); const [isOpen, setIsOpen] = useState(false); - const placement = 'bottom-end'; - // the order of middleware is important! - const middleware = [offset(0), ...getPositioningMiddleware(placement)]; - - const { context, refs, floatingStyles } = useFloating({ - open: isOpen, - placement, - onOpenChange: setIsOpen, - middleware, - whileElementsMounted: autoUpdate, - }); - - const click = useClick(context); - const dismiss = useDismiss(context); - - const { getReferenceProps, getFloatingProps } = useInteractions([dismiss, click]); - - const onChangeInternal = (item: SelectableValue) => { - onChange(item); - setIsOpen(false); - }; + const renderMenu = () => ( + setIsOpen(false)}> + + {options.map((item) => ( + onChange(item)} + active={item.value === value?.value} + ariaChecked={item.value === value?.value} + ariaLabel={item.ariaLabel || item.label} + disabled={item.isDisabled} + component={item.component} + role="menuitemradio" + /> + ))} + + + ); return ( -
- + + {value?.label || (value?.value != null ? String(value?.value) : null)} - {isOpen && ( -
- - {/* - tabIndex=-1 is needed here to support highlighting text within the menu when using FocusScope - see https://github.com/adobe/react-spectrum/issues/1604#issuecomment-781574668 - */} - setIsOpen(false)}> - {options.map((item) => ( - onChangeInternal(item)} - active={item.value === value?.value} - ariaChecked={item.value === value?.value} - ariaLabel={item.ariaLabel || item.label} - disabled={item.isDisabled} - component={item.component} - role="menuitemradio" - /> - ))} - - -
- )} -
+ ); }; @@ -102,17 +64,3 @@ ButtonSelectComponent.displayName = 'ButtonSelect'; // see https://github.com/DefinitelyTyped/DefinitelyTyped/issues/37087#issuecomment-656596623 // eslint-disable-next-line @typescript-eslint/consistent-type-assertions export const ButtonSelect = memo(ButtonSelectComponent) as typeof ButtonSelectComponent; - -const getStyles = (theme: GrafanaTheme2) => { - return { - wrapper: css({ - position: 'relative', - display: 'inline-flex', - }), - menuWrapper: css({ - zIndex: theme.zIndex.dropdown, - maxHeight: '100vh', - overflow: 'auto', - }), - }; -}; From 172e040065bb85e255ded8847f54b744df25f42c Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 22 Sep 2025 21:59:34 +0300 Subject: [PATCH 20/54] Live: remove `annotations.Repository` (#111433) --- pkg/api/dashboard_test.go | 2 -- pkg/server/wire_gen.go | 8 ++++---- pkg/services/live/live.go | 4 +--- pkg/services/live/live_test.go | 2 -- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/api/dashboard_test.go b/pkg/api/dashboard_test.go index cdf85c8594d..7c47656e208 100644 --- a/pkg/api/dashboard_test.go +++ b/pkg/api/dashboard_test.go @@ -31,7 +31,6 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" "github.com/grafana/grafana/pkg/services/accesscontrol/actest" - "github.com/grafana/grafana/pkg/services/annotations/annotationstest" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" dashver "github.com/grafana/grafana/pkg/services/dashboardversion" @@ -129,7 +128,6 @@ func newTestLive(t *testing.T, store db.DB) *live.GrafanaLive { nil, features, acimpl.ProvideAccessControl(features), &dashboards.FakeDashboardService{}, - annotationstest.NewFakeAnnotationsRepo(), nil, nil) require.NoError(t, err) return gLive diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index eba4ca46baf..fa09706c01c 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -644,8 +644,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api qsDatasourceClientBuilder := dsquerierclient.NewNullQSDatasourceClientBuilder() exprService := expr.ProvideService(cfg, middlewareHandler, plugincontextProvider, featureToggles, registerer, tracingService, qsDatasourceClientBuilder) queryServiceImpl := query.ProvideService(cfg, cacheServiceImpl, exprService, ossDataSourceRequestValidator, middlewareHandler, plugincontextProvider, qsDatasourceClientBuilder) - repositoryImpl := annotationsimpl.ProvideService(sqlStore, cfg, featureToggles, tagimplService, tracingService, dBstore, dashboardService, registerer) - grafanaLive, err := live.ProvideService(plugincontextProvider, cfg, routeRegisterImpl, pluginstoreService, middlewareHandler, cacheService, cacheServiceImpl, sqlStore, secretsService, usageStats, queryServiceImpl, featureToggles, accessControl, dashboardService, repositoryImpl, orgService, eventualRestConfigProvider) + grafanaLive, err := live.ProvideService(plugincontextProvider, cfg, routeRegisterImpl, pluginstoreService, middlewareHandler, cacheService, cacheServiceImpl, sqlStore, secretsService, usageStats, queryServiceImpl, featureToggles, accessControl, dashboardService, orgService, eventualRestConfigProvider) if err != nil { return nil, err } @@ -655,6 +654,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api contexthandlerContextHandler := contexthandler.ProvideService(cfg, authnAuthenticator, featureToggles) logger := loggermw.Provide(cfg, featureToggles) ngAlert := metrics2.ProvideService() + repositoryImpl := annotationsimpl.ProvideService(sqlStore, cfg, featureToggles, tagimplService, tracingService, dBstore, dashboardService, registerer) alertNG, err := ngalert.ProvideService(cfg, featureToggles, cacheServiceImpl, service15, routeRegisterImpl, sqlStore, kvStore, exprService, dataSourceProxyService, quotaService, secretsService, notificationService, ngAlert, folderimplService, accessControl, dashboardService, renderingService, inProcBus, acimplService, repositoryImpl, pluginstoreService, tracingService, dBstore, httpclientProvider, plugincontextProvider, receiverPermissionsService, userService) if err != nil { return nil, err @@ -1248,8 +1248,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac qsDatasourceClientBuilder := dsquerierclient.NewNullQSDatasourceClientBuilder() exprService := expr.ProvideService(cfg, middlewareHandler, plugincontextProvider, featureToggles, registerer, tracingService, qsDatasourceClientBuilder) queryServiceImpl := query.ProvideService(cfg, cacheServiceImpl, exprService, ossDataSourceRequestValidator, middlewareHandler, plugincontextProvider, qsDatasourceClientBuilder) - repositoryImpl := annotationsimpl.ProvideService(sqlStore, cfg, featureToggles, tagimplService, tracingService, dBstore, dashboardService, registerer) - grafanaLive, err := live.ProvideService(plugincontextProvider, cfg, routeRegisterImpl, pluginstoreService, middlewareHandler, cacheService, cacheServiceImpl, sqlStore, secretsService, usageStats, queryServiceImpl, featureToggles, accessControl, dashboardService, repositoryImpl, orgService, eventualRestConfigProvider) + grafanaLive, err := live.ProvideService(plugincontextProvider, cfg, routeRegisterImpl, pluginstoreService, middlewareHandler, cacheService, cacheServiceImpl, sqlStore, secretsService, usageStats, queryServiceImpl, featureToggles, accessControl, dashboardService, orgService, eventualRestConfigProvider) if err != nil { return nil, err } @@ -1260,6 +1259,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac logger := loggermw.Provide(cfg, featureToggles) notificationServiceMock := notifications.MockNotificationService() ngAlert := metrics2.ProvideServiceForTest() + repositoryImpl := annotationsimpl.ProvideService(sqlStore, cfg, featureToggles, tagimplService, tracingService, dBstore, dashboardService, registerer) alertNG, err := ngalert.ProvideService(cfg, featureToggles, cacheServiceImpl, service15, routeRegisterImpl, sqlStore, kvStore, exprService, dataSourceProxyService, quotaService, secretsService, notificationServiceMock, ngAlert, folderimplService, accessControl, dashboardService, renderingService, inProcBus, acimplService, repositoryImpl, pluginstoreService, tracingService, dBstore, httpclientProvider, plugincontextProvider, receiverPermissionsService, userService) if err != nil { return nil, err diff --git a/pkg/services/live/live.go b/pkg/services/live/live.go index 0394e06fdc0..d3648d12f5c 100644 --- a/pkg/services/live/live.go +++ b/pkg/services/live/live.go @@ -25,7 +25,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/live" - "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/api/routing" @@ -39,7 +38,6 @@ import ( "github.com/grafana/grafana/pkg/middleware/requestmeta" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/annotations" "github.com/grafana/grafana/pkg/services/apiserver" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" @@ -84,7 +82,7 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r pluginStore pluginstore.Store, pluginClient plugins.Client, cacheService *localcache.CacheService, dataSourceCache datasources.CacheService, sqlStore db.DB, secretsService secrets.Service, usageStatsService usagestats.Service, queryDataService query.Service, toggles featuremgmt.FeatureToggles, - accessControl accesscontrol.AccessControl, dashboardService dashboards.DashboardService, annotationsRepo annotations.Repository, + accessControl accesscontrol.AccessControl, dashboardService dashboards.DashboardService, orgService org.Service, configProvider apiserver.RestConfigProvider) (*GrafanaLive, error) { g := &GrafanaLive{ Cfg: cfg, diff --git a/pkg/services/live/live_test.go b/pkg/services/live/live_test.go index cfead7e50ec..00ce4bee3f4 100644 --- a/pkg/services/live/live_test.go +++ b/pkg/services/live/live_test.go @@ -22,7 +22,6 @@ import ( "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" - "github.com/grafana/grafana/pkg/services/annotations/annotationstest" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/live/livecontext" @@ -355,7 +354,6 @@ func setupLiveService(cfg *setting.Cfg, t *testing.T) (*GrafanaLive, error) { featuremgmt.WithFeatures(), acimpl.ProvideAccessControl(featuremgmt.WithFeatures()), &dashboards.FakeDashboardService{}, - annotationstest.NewFakeAnnotationsRepo(), nil, nil) } From b02d2762f57a4eea7106cb1f404c4dba983a5662 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Mon, 22 Sep 2025 13:05:34 -0600 Subject: [PATCH 21/54] Provisioning: Add metrics to jobs (#111447) --- pkg/operators/provisioning/jobs_operator.go | 10 +- .../apis/provisioning/jobs/delete/worker.go | 25 +++- .../provisioning/jobs/delete/worker_test.go | 54 +++++---- .../apis/provisioning/jobs/export/worker.go | 31 ++++- .../provisioning/jobs/export/worker_test.go | 42 ++++--- .../apis/provisioning/jobs/metrics.go | 112 ++++++++++++++++++ .../apis/provisioning/jobs/move/worker.go | 25 +++- .../provisioning/jobs/move/worker_test.go | 48 ++++---- .../apis/provisioning/jobs/persistentstore.go | 16 ++- .../apis/provisioning/jobs/sync/worker.go | 32 ++++- .../provisioning/jobs/sync/worker_test.go | 9 +- pkg/registry/apis/provisioning/register.go | 10 +- 12 files changed, 316 insertions(+), 98 deletions(-) create mode 100644 pkg/registry/apis/provisioning/jobs/metrics.go diff --git a/pkg/operators/provisioning/jobs_operator.go b/pkg/operators/provisioning/jobs_operator.go index 49fa2676717..e108e8a8d3d 100644 --- a/pkg/operators/provisioning/jobs_operator.go +++ b/pkg/operators/provisioning/jobs_operator.go @@ -174,6 +174,8 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis workers := make([]jobs.Worker, 0) + metrics := jobs.RegisterJobMetrics(registry) + // Sync syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync) syncWorker := sync.NewSyncWorker( @@ -182,7 +184,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis nil, // HACK: we have updated the worker to check for nil statusPatcher.Patch, syncer, - registry, + metrics, ) workers = append(workers, syncWorker) @@ -193,7 +195,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis repositoryResources, export.ExportAll, stageIfPossible, - registry, + metrics, ) workers = append(workers, exportWorker) @@ -208,11 +210,11 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis workers = append(workers, migrationWorker) // Delete - deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, repositoryResources, registry) + deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, repositoryResources, metrics) workers = append(workers, deleteWorker) // Move - moveWorker := move.NewWorker(syncWorker, stageIfPossible, repositoryResources, registry) + moveWorker := move.NewWorker(syncWorker, stageIfPossible, repositoryResources, metrics) workers = append(workers, moveWorker) return workers, nil diff --git a/pkg/registry/apis/provisioning/jobs/delete/worker.go b/pkg/registry/apis/provisioning/jobs/delete/worker.go index 738876d1517..dd627f4184c 100644 --- a/pkg/registry/apis/provisioning/jobs/delete/worker.go +++ b/pkg/registry/apis/provisioning/jobs/delete/worker.go @@ -8,26 +8,26 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" - "github.com/prometheus/client_golang/prometheus" ) type Worker struct { syncWorker jobs.Worker wrapFn repository.WrapWithStageFn resourcesFactory resources.RepositoryResourcesFactory - registry prometheus.Registerer + metrics jobs.JobMetrics } -func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory, registry prometheus.Registerer) *Worker { +func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory, metrics jobs.JobMetrics) *Worker { return &Worker{ syncWorker: syncWorker, wrapFn: wrapFn, resourcesFactory: resourcesFactory, - registry: registry, + metrics: metrics, } } @@ -40,8 +40,15 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr return errors.New("missing delete settings") } + logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) opts := *job.Spec.Delete paths := opts.Paths + start := time.Now() + outcome := jobs.ErrorOutcome + resourcesDeleted := 0 + defer func() { + w.metrics.RecordJob(string(provisioning.JobActionDelete), outcome, resourcesDeleted, time.Since(start).Seconds()) + }() progress.SetTotal(ctx, len(paths)+len(opts.Resources)) progress.StrictMaxErrors(1) // Fail fast on any error during deletion @@ -49,6 +56,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr fn := func(repo repository.Repository, _ bool) error { rw, ok := repo.(repository.ReaderWriter) if !ok { + logger.Error("delete job submitted targeting repository that is not a ReaderWriter") return errors.New("delete job submitted targeting repository that is not a ReaderWriter") } @@ -56,6 +64,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr if len(opts.Resources) > 0 { resolvedPaths, err := w.resolveResourcesToPaths(ctx, rw, progress, opts.Resources) if err != nil { + logger.Error("failed to resolve resource paths", "error", err) return err } paths = append(paths, resolvedPaths...) @@ -78,6 +87,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr err := w.wrapFn(ctx, repo, stageOptions, fn) if err != nil { + logger.Error("failed to delete files from repository", "error", err) return fmt.Errorf("delete files from repository: %w", err) } @@ -104,10 +114,17 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr } if err := w.syncWorker.Process(ctx, repo, syncJob, progress); err != nil { + logger.Error("failed to pull resources", "error", err) return fmt.Errorf("pull resources: %w", err) } } + outcome = jobs.SuccessOutcome + jobStatus := progress.Complete(ctx, nil) + for _, summary := range jobStatus.Summary { + resourcesDeleted += int(summary.Delete) + } + return nil } diff --git a/pkg/registry/apis/provisioning/jobs/delete/worker_test.go b/pkg/registry/apis/provisioning/jobs/delete/worker_test.go index c5fd53282f8..07cc855f194 100644 --- a/pkg/registry/apis/provisioning/jobs/delete/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/delete/worker_test.go @@ -38,6 +38,7 @@ func (s *simpleRepository) Test(ctx context.Context) (*v0alpha1.TestResults, err } func TestDeleteWorker_IsSupported(t *testing.T) { + metrics := jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()) tests := []struct { name string job v0alpha1.Job @@ -74,7 +75,7 @@ func TestDeleteWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, metrics) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -88,7 +89,7 @@ func TestDeleteWorker_ProcessMissingDeleteSettings(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing delete settings") } @@ -117,7 +118,7 @@ func TestDeleteWorker_ProcessNotReaderWriter(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: delete job submitted targeting repository that is not a ReaderWriter") } @@ -140,7 +141,7 @@ func TestDeleteWorker_ProcessWrapFnError(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: stage failed") } @@ -177,6 +178,7 @@ func TestDeleteWorker_ProcessDeleteFilesSuccess(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Deleting test/path1").Return() mockProgress.On("SetMessage", mock.Anything, "Deleting test/path2").Return() mockProgress.On("TooManyErrors").Return(nil).Twice() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) mockRepo.On("Delete", mock.Anything, "test/path1", "main", "Delete test/path1").Return(nil) mockRepo.On("Delete", mock.Anything, "test/path2", "main", "Delete test/path2").Return(nil) @@ -188,7 +190,7 @@ func TestDeleteWorker_ProcessDeleteFilesSuccess(t *testing.T) { return result.Path == "test/path2" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -226,7 +228,7 @@ func TestDeleteWorker_ProcessDeleteFilesWithError(t *testing.T) { })).Return() mockProgress.On("TooManyErrors").Return(errors.New("too many errors")) - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: too many errors") } @@ -256,6 +258,7 @@ func TestDeleteWorker_ProcessWithSyncWorker(t *testing.T) { mockProgress.On("StrictMaxErrors", 1).Return() mockProgress.On("SetMessage", mock.Anything, "Deleting test/path").Return() mockProgress.On("TooManyErrors").Return(nil) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) mockRepo.On("Delete", mock.Anything, "test/path", "", "Delete test/path").Return(nil) @@ -270,7 +273,7 @@ func TestDeleteWorker_ProcessWithSyncWorker(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -310,7 +313,7 @@ func TestDeleteWorker_ProcessSyncWorkerError(t *testing.T) { syncError := errors.New("sync failed") mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.Anything, mockProgress).Return(syncError) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "pull resources: sync failed") } @@ -379,7 +382,7 @@ func TestDeleteWorker_deleteFiles(t *testing.T) { } } - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.deleteFiles(context.Background(), mockRepo, mockProgress, opts, tt.paths...) if tt.expectedError != "" { @@ -457,6 +460,7 @@ func TestDeleteWorker_ProcessWithResourceRefs(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Deleting dashboards/test-dashboard.json").Return() mockProgress.On("SetMessage", mock.Anything, "Deleting folders/test-folder.json").Return() mockProgress.On("TooManyErrors").Return(nil).Times(3) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Mock file deletions mockRepo.On("Delete", mock.Anything, "test/path1", "main", "Delete test/path1").Return(nil) @@ -474,7 +478,7 @@ func TestDeleteWorker_ProcessWithResourceRefs(t *testing.T) { return result.Path == "folders/test-folder.json" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) @@ -525,6 +529,7 @@ func TestDeleteWorker_ProcessResourceRefsOnly(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Finding path for resource dashboard.grafana.app/Dashboard/test-dashboard").Return() mockProgress.On("SetMessage", mock.Anything, "Deleting dashboards/test-dashboard.json").Return() mockProgress.On("TooManyErrors").Return(nil) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) mockRepo.On("Delete", mock.Anything, "dashboards/test-dashboard.json", "main", "Delete dashboards/test-dashboard.json").Return(nil) @@ -532,7 +537,7 @@ func TestDeleteWorker_ProcessResourceRefsOnly(t *testing.T) { return result.Path == "dashboards/test-dashboard.json" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -578,6 +583,7 @@ func TestDeleteWorker_ProcessResourceResolutionError(t *testing.T) { mockProgress.On("StrictMaxErrors", 1).Return() mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return() mockProgress.On("SetMessage", mock.Anything, "Finding path for resource dashboard.grafana.app/Dashboard/nonexistent-dashboard").Return() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Expect error to be recorded, not thrown mockProgress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { @@ -597,7 +603,7 @@ func TestDeleteWorker_ProcessResourceResolutionError(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) // Should succeed even with resource resolution error } @@ -636,7 +642,7 @@ func TestDeleteWorker_ProcessResourcesFactoryError(t *testing.T) { mockProgress.On("StrictMaxErrors", 1).Return() mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: create repository resources client: failed to create repository resources client") } @@ -672,7 +678,7 @@ func TestDeleteWorker_ProcessResourceRefsNotReaderWriter(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: delete job submitted targeting repository that is not a ReaderWriter") } @@ -725,7 +731,7 @@ func TestDeleteWorker_ProcessResourceResolutionTooManyErrors(t *testing.T) { })).Return() mockProgress.On("TooManyErrors").Return(errors.New("too many errors")) - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "delete files from repository: too many errors") } @@ -801,7 +807,7 @@ func TestDeleteWorker_ProcessMixedResourcesWithPartialFailure(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Finding path for resource folder.grafana.app/Folder/valid-folder").Return() mockProgress.On("SetMessage", mock.Anything, "Deleting dashboards/valid-dashboard.json").Return() mockProgress.On("SetMessage", mock.Anything, "Deleting folders/valid-folder.json").Return() - + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Record the error for the failed resource mockProgress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool { return result.Name == "nonexistent-dashboard" && result.Error != nil @@ -822,7 +828,7 @@ func TestDeleteWorker_ProcessMixedResourcesWithPartialFailure(t *testing.T) { return result.Path == "folders/valid-folder.json" && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) // Should succeed overall, with only the failed resource recorded as error } @@ -875,7 +881,7 @@ func TestDeleteWorker_ProcessWithPathDeduplication(t *testing.T) { // Expect total of 5 items (2 explicit paths + 3 resources), but only 3 unique paths will be deleted mockProgress.On("SetTotal", mock.Anything, 5).Return() mockProgress.On("StrictMaxErrors", 1).Return() - + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Resource resolution phase mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return() @@ -920,7 +926,7 @@ func TestDeleteWorker_ProcessWithPathDeduplication(t *testing.T) { return result.Path == "dashboards/unique-dashboard.json" && result.Action == repository.FileActionDeleted && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) @@ -1015,7 +1021,7 @@ func TestDeleteWorker_RefURLsSetWithRef(t *testing.T) { mockProgress.On("Record", mock.Anything, mock.Anything).Once() mockProgress.On("TooManyErrors").Return(nil).Once() mockProgress.On("SetRefURLs", mock.Anything, expectedRefURLs).Once() - + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) mockReaderWriter := repository.NewMockReaderWriter(t) mockReaderWriter.On("Delete", mock.Anything, "test.json", "feature-branch", "Delete test.json").Return(nil) @@ -1037,7 +1043,7 @@ func TestDeleteWorker_RefURLsSetWithRef(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -1067,6 +1073,7 @@ func TestDeleteWorker_RefURLsNotSetWithoutRef(t *testing.T) { mockProgress.On("TooManyErrors").Return(nil).Once() mockProgress.On("ResetResults").Once() mockProgress.On("SetMessage", mock.Anything, "pull resources").Once() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // SetRefURLs should NOT be called since no ref is specified mockReaderWriter := repository.NewMockReaderWriter(t) @@ -1093,7 +1100,7 @@ func TestDeleteWorker_RefURLsNotSetWithoutRef(t *testing.T) { }, } - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -1120,6 +1127,7 @@ func TestDeleteWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Deleting test.json").Once() mockProgress.On("Record", mock.Anything, mock.Anything).Once() mockProgress.On("TooManyErrors").Return(nil).Once() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // SetRefURLs should NOT be called since repo doesn't support URLs mockReaderWriter := repository.NewMockReaderWriter(t) @@ -1143,7 +1151,7 @@ func TestDeleteWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) diff --git a/pkg/registry/apis/provisioning/jobs/export/worker.go b/pkg/registry/apis/provisioning/jobs/export/worker.go index f7def54ac1f..8835a7b6faa 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker.go @@ -6,11 +6,11 @@ import ( "fmt" "time" + "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" - "github.com/prometheus/client_golang/prometheus" ) //go:generate mockery --name ExportFn --structname MockExportFn --inpackage --filename mock_export_fn.go --with-expecter @@ -24,7 +24,7 @@ type ExportWorker struct { repositoryResources resources.RepositoryResourcesFactory exportFn ExportFn wrapWithStageFn WrapWithStageFn - registry prometheus.Registerer + metrics jobs.JobMetrics } func NewExportWorker( @@ -32,14 +32,14 @@ func NewExportWorker( repositoryResources resources.RepositoryResourcesFactory, exportFn ExportFn, wrapWithStageFn WrapWithStageFn, - registry prometheus.Registerer, + metrics jobs.JobMetrics, ) *ExportWorker { return &ExportWorker{ clientFactory: clientFactory, repositoryResources: repositoryResources, exportFn: exportFn, wrapWithStageFn: wrapWithStageFn, - registry: registry, + metrics: metrics, } } @@ -54,6 +54,13 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, return errors.New("missing export settings") } + logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) + start := time.Now() + outcome := jobs.ErrorOutcome + resourcesExported := 0 + defer func() { + r.metrics.RecordJob(string(provisioning.JobActionPush), outcome, resourcesExported, time.Since(start).Seconds()) + }() cfg := repo.Config() // Can write to external branch if err := repository.IsWriteAllowed(cfg, options.Branch); err != nil { @@ -76,16 +83,19 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, fn := func(repo repository.Repository, _ bool) error { clients, err := r.clientFactory.Clients(ctx, cfg.Namespace) if err != nil { + logger.Error("failed to create clients", "error", err) return fmt.Errorf("create clients: %w", err) } rw, ok := repo.(repository.ReaderWriter) if !ok { + logger.Error("export job submitted targeting repository that is not a ReaderWriter") return errors.New("export job submitted targeting repository that is not a ReaderWriter") } repositoryResources, err := r.repositoryResources.Client(ctx, rw) if err != nil { + logger.Error("failed to create repository resource client", "error", err) return fmt.Errorf("create repository resource client: %w", err) } @@ -103,5 +113,16 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, } } - return err + if err != nil { + logger.Error("failed to export", "error", err) + return err + } + + outcome = jobs.SuccessOutcome + jobStatus := progress.Complete(ctx, nil) + for _, summary := range jobStatus.Summary { + resourcesExported += int(summary.Write) + } + + return nil } diff --git a/pkg/registry/apis/provisioning/jobs/export/worker_test.go b/pkg/registry/apis/provisioning/jobs/export/worker_test.go index 88b69691667..c5bf32a62b6 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker_test.go @@ -19,6 +19,7 @@ import ( ) func TestExportWorker_IsSupported(t *testing.T) { + metrics := jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()) tests := []struct { name string job v0alpha1.Job @@ -55,7 +56,7 @@ func TestExportWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) + r := NewExportWorker(nil, nil, nil, nil, metrics) got := r.IsSupported(context.Background(), tt.job) require.Equal(t, tt.want, got) }) @@ -69,7 +70,7 @@ func TestExportWorker_ProcessNoExportSettings(t *testing.T) { }, } - r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) + r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing export settings") } @@ -92,7 +93,7 @@ func TestExportWorker_ProcessWriteNotAllowed(t *testing.T) { }, }) - r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) + r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, nil) require.EqualError(t, err, "this repository is read only") } @@ -116,7 +117,7 @@ func TestExportWorker_ProcessBranchNotAllowedForLocal(t *testing.T) { }, }) - r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) + r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, nil) require.EqualError(t, err, "this repository does not support the branch workflow") } @@ -148,7 +149,7 @@ func TestExportWorker_ProcessFailedToCreateClients(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) mockProgress := jobs.NewMockJobProgressRecorder(t) err := r.Process(context.Background(), mockRepo, job, mockProgress) @@ -184,7 +185,7 @@ func TestExportWorker_ProcessNotReaderWriter(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export job submitted targeting repository that is not a ReaderWriter") } @@ -220,7 +221,7 @@ func TestExportWorker_ProcessRepositoryResourcesError(t *testing.T) { mockStageFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOpts repository.StageOptions, fn func(repository.Repository, bool) error) error { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "create repository resource client: failed to create repository resources client") } @@ -248,6 +249,7 @@ func TestExportWorker_ProcessStageOptions(t *testing.T) { }) mockProgress := jobs.NewMockJobProgressRecorder(t) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // No progress messages expected in current implementation mockClients := resources.NewMockClientFactory(t) @@ -271,7 +273,7 @@ func TestExportWorker_ProcessStageOptions(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -331,6 +333,7 @@ func TestExportWorker_ProcessStageOptionsWithBranch(t *testing.T) { }) mockProgress := jobs.NewMockJobProgressRecorder(t) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) mockClients := resources.NewMockClientFactory(t) mockResourceClients := resources.NewMockResourceClients(t) mockClients.On("Clients", mock.Anything, "test-namespace").Return(mockResourceClients, nil) @@ -352,7 +355,7 @@ func TestExportWorker_ProcessStageOptionsWithBranch(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) }) @@ -395,7 +398,7 @@ func TestExportWorker_ProcessExportFnError(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export failed") } @@ -423,7 +426,7 @@ func TestExportWorker_ProcessWrapWithStageFnError(t *testing.T) { mockStageFn := NewMockWrapWithStageFn(t) mockStageFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(errors.New("stage failed")) - r := NewExportWorker(nil, nil, nil, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(nil, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "stage failed") } @@ -449,7 +452,7 @@ func TestExportWorker_ProcessBranchNotAllowedForStageableRepositories(t *testing mockProgress := jobs.NewMockJobProgressRecorder(t) // No progress messages expected in current implementation - r := NewExportWorker(nil, nil, nil, nil, prometheus.DefaultRegisterer) + r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "this repository does not support the branch workflow") } @@ -479,6 +482,7 @@ func TestExportWorker_ProcessGitRepository(t *testing.T) { }) mockProgress := jobs.NewMockJobProgressRecorder(t) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // No progress messages expected in current implementation mockClients := resources.NewMockClientFactory(t) @@ -500,7 +504,7 @@ func TestExportWorker_ProcessGitRepository(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -546,7 +550,7 @@ func TestExportWorker_ProcessGitRepositoryExportFnError(t *testing.T) { return fn(repo, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "export failed") } @@ -587,7 +591,7 @@ func TestExportWorker_RefURLsSetWithBranch(t *testing.T) { // Mock progress recorder to expect SetRefURLs call mockProgress := jobs.NewMockJobProgressRecorder(t) mockProgress.On("SetRefURLs", mock.Anything, expectedRefURLs).Once() - + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Mock other dependencies mockClients := resources.NewMockClientFactory(t) mockResourceClients := resources.NewMockResourceClients(t) @@ -609,7 +613,7 @@ func TestExportWorker_RefURLsSetWithBranch(t *testing.T) { return fn(mockReaderWriter, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -644,6 +648,7 @@ func TestExportWorker_RefURLsNotSetWithoutBranch(t *testing.T) { // Mock progress recorder - SetRefURLs should NOT be called mockProgress := jobs.NewMockJobProgressRecorder(t) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Explicitly NOT expecting SetRefURLs call // Mock other dependencies @@ -665,7 +670,7 @@ func TestExportWorker_RefURLsNotSetWithoutBranch(t *testing.T) { return fn(mockReaderWriter, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -700,6 +705,7 @@ func TestExportWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { // Mock progress recorder - SetRefURLs should NOT be called mockProgress := jobs.NewMockJobProgressRecorder(t) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{}) // Explicitly NOT expecting SetRefURLs call // Mock other dependencies @@ -721,7 +727,7 @@ func TestExportWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { return fn(mockReaderWriter, true) }) - r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, prometheus.DefaultRegisterer) + r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := r.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) diff --git a/pkg/registry/apis/provisioning/jobs/metrics.go b/pkg/registry/apis/provisioning/jobs/metrics.go new file mode 100644 index 00000000000..8a4a8d20eb6 --- /dev/null +++ b/pkg/registry/apis/provisioning/jobs/metrics.go @@ -0,0 +1,112 @@ +package jobs + +import "github.com/prometheus/client_golang/prometheus" + +const ( + SuccessOutcome = "success" + ErrorOutcome = "error" +) + +type JobMetrics struct { + registry prometheus.Registerer + processedTotal *prometheus.CounterVec + durationHist *prometheus.HistogramVec +} + +type QueueMetrics struct { + queueSize *prometheus.GaugeVec + queueWaitTime *prometheus.HistogramVec +} + +func RegisterQueueMetrics(registry prometheus.Registerer) QueueMetrics { + queueSize := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "grafana_provisioning_jobs_queue_size", + Help: "Number of jobs currently in the queue", + }, + []string{"action"}, + ) + registry.MustRegister(queueSize) + + queueWaitTime := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grafana_provisioning_jobs_queue_wait_seconds", + Help: "Time jobs spend waiting in the queue before being claimed", + Buckets: []float64{1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0}, + }, + []string{"action"}, + ) + registry.MustRegister(queueWaitTime) + + return QueueMetrics{ + queueSize: queueSize, + queueWaitTime: queueWaitTime, + } +} + +func (m *QueueMetrics) IncreaseQueueSize(action string) { + m.queueSize.WithLabelValues(action).Inc() +} + +func (m *QueueMetrics) DecreaseQueueSize(action string) { + m.queueSize.WithLabelValues(action).Dec() +} + +func (m *QueueMetrics) RecordWaitTime(action string, waitSeconds float64) { + m.queueWaitTime.WithLabelValues(action).Observe(waitSeconds) +} + +func RegisterJobMetrics(registry prometheus.Registerer) JobMetrics { + processedTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "grafana_provisioning_jobs_processed_total", + Help: "Total number of jobs processed", + }, + []string{"action", "outcome"}, + ) + registry.MustRegister(processedTotal) + + durationHist := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grafana_provisioning_jobs_duration_seconds", + Help: "Duration of job", + Buckets: []float64{5.0, 10.0, 30.0, 60.0, 120.0, 300.0}, + }, + []string{"action", "resources_changed_bucket"}, + ) + registry.MustRegister(durationHist) + + return JobMetrics{ + registry: registry, + processedTotal: processedTotal, + durationHist: durationHist, + } +} + +func (m *JobMetrics) RecordJob(jobAction string, outcome string, resourceCountChanged int, duration float64) { + m.processedTotal.WithLabelValues(jobAction, outcome).Inc() + + // only record duration when the job was successful. otherwise resource count will be incorrect + if outcome == SuccessOutcome { + m.durationHist.WithLabelValues(jobAction, getResourceCountBucket(resourceCountChanged)).Observe(duration) + } +} + +func getResourceCountBucket(count int) string { + switch { + case count == 0: + return "0" + case count <= 10: + return "1-10" + case count <= 50: + return "11-50" + case count <= 100: + return "51-100" + case count <= 500: + return "101-500" + case count <= 1000: + return "501-1000" + default: + return "1000+" + } +} diff --git a/pkg/registry/apis/provisioning/jobs/move/worker.go b/pkg/registry/apis/provisioning/jobs/move/worker.go index 35c4e6b0342..6123bad5278 100644 --- a/pkg/registry/apis/provisioning/jobs/move/worker.go +++ b/pkg/registry/apis/provisioning/jobs/move/worker.go @@ -9,27 +9,27 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/apps/provisioning/pkg/safepath" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" - "github.com/prometheus/client_golang/prometheus" ) type Worker struct { syncWorker jobs.Worker wrapFn repository.WrapWithStageFn resourcesFactory resources.RepositoryResourcesFactory - registry prometheus.Registerer + metrics jobs.JobMetrics } -func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory, registry prometheus.Registerer) *Worker { +func NewWorker(syncWorker jobs.Worker, wrapFn repository.WrapWithStageFn, resourcesFactory resources.RepositoryResourcesFactory, metrics jobs.JobMetrics) *Worker { return &Worker{ syncWorker: syncWorker, wrapFn: wrapFn, resourcesFactory: resourcesFactory, - registry: registry, + metrics: metrics, } } @@ -42,6 +42,13 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr return errors.New("missing move settings") } opts := *job.Spec.Move + logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) + outcome := jobs.ErrorOutcome + start := time.Now() + resourcesMoved := 0 + defer func() { + w.metrics.RecordJob(string(provisioning.JobActionMove), outcome, resourcesMoved, time.Since(start).Seconds()) + }() if opts.TargetPath == "" { return errors.New("target path is required for move operation") @@ -60,6 +67,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr fn := func(repo repository.Repository, _ bool) error { rw, ok := repo.(repository.ReaderWriter) if !ok { + logger.Error("move job submitted targeting repository that is not a ReaderWriter") return errors.New("move job submitted targeting repository that is not a ReaderWriter") } @@ -89,6 +97,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr err := w.wrapFn(ctx, repo, stageOptions, fn) if err != nil { + logger.Error("failed to move files in repository", "error", err) return fmt.Errorf("move files in repository: %w", err) } @@ -115,10 +124,18 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr } if err := w.syncWorker.Process(ctx, repo, syncJob, progress); err != nil { + logger.Error("failed to pull resources", "error", err) return fmt.Errorf("pull resources: %w", err) } } + outcome = jobs.SuccessOutcome + jobStatus := progress.Complete(ctx, nil) + for _, summary := range jobStatus.Summary { + // FileActionRenamed increments both delete & create, use create here + resourcesMoved += int(summary.Create) + } + return nil } diff --git a/pkg/registry/apis/provisioning/jobs/move/worker_test.go b/pkg/registry/apis/provisioning/jobs/move/worker_test.go index f8f82f20142..f5129350e6e 100644 --- a/pkg/registry/apis/provisioning/jobs/move/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/move/worker_test.go @@ -31,6 +31,7 @@ func (m *mockReaderWriter) Move(ctx context.Context, oldPath, newPath, ref, mess } func TestMoveWorker_IsSupported(t *testing.T) { + metrics := jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()) tests := []struct { name string job provisioning.Job @@ -76,7 +77,7 @@ func TestMoveWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, metrics) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -90,7 +91,7 @@ func TestMoveWorker_ProcessMissingMoveSettings(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "missing move settings") } @@ -105,7 +106,7 @@ func TestMoveWorker_ProcessMissingTargetPath(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "target path is required for move operation") } @@ -121,7 +122,7 @@ func TestMoveWorker_ProcessInvalidTargetPath(t *testing.T) { }, } - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), nil, job, nil) require.EqualError(t, err, "target path must be a directory (should end with '/')") } @@ -153,7 +154,7 @@ func TestMoveWorker_ProcessNotReaderWriter(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: move job submitted targeting repository that is not a ReaderWriter") } @@ -177,7 +178,7 @@ func TestMoveWorker_ProcessWrapFnError(t *testing.T) { mockProgress.On("SetTotal", mock.Anything, 1).Return() mockProgress.On("StrictMaxErrors", 1).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: stage failed") } @@ -213,6 +214,7 @@ func TestMoveWorker_ProcessMoveFilesSuccess(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Moving test/path1 to new/location/path1").Return() mockProgress.On("SetMessage", mock.Anything, "Moving test/path2 to new/location/path2").Return() mockProgress.On("TooManyErrors").Return(nil).Twice() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) mockRepo.On("Move", mock.Anything, "test/path1", "new/location/path1", "main", "Move test/path1 to new/location/path1").Return(nil) mockRepo.On("Move", mock.Anything, "test/path2", "new/location/path2", "main", "Move test/path2 to new/location/path2").Return(nil) @@ -224,7 +226,7 @@ func TestMoveWorker_ProcessMoveFilesSuccess(t *testing.T) { return result.Path == "test/path2" && result.Action == repository.FileActionRenamed && result.Error == nil })).Return() - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -263,7 +265,7 @@ func TestMoveWorker_ProcessMoveFilesWithError(t *testing.T) { })).Return() mockProgress.On("TooManyErrors").Return(errors.New("too many errors")) - worker := NewWorker(nil, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: too many errors") } @@ -303,12 +305,13 @@ func TestMoveWorker_ProcessWithSyncWorker(t *testing.T) { mockProgress.On("ResetResults").Return() mockProgress.On("SetMessage", mock.Anything, "pull resources").Return() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.MatchedBy(func(syncJob provisioning.Job) bool { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -349,7 +352,7 @@ func TestMoveWorker_ProcessSyncWorkerError(t *testing.T) { syncError := errors.New("sync failed") mockSyncWorker.On("Process", mock.Anything, mockRepo, mock.Anything, mockProgress).Return(syncError) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "pull resources: sync failed") } @@ -430,7 +433,7 @@ func TestMoveWorker_moveFiles(t *testing.T) { } } - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.moveFiles(context.Background(), mockRepo, mockProgress, opts, tt.paths...) if tt.expectedError != "" { @@ -486,7 +489,7 @@ func TestMoveWorker_constructTargetPath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewWorker(nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) result := worker.constructTargetPath(tt.jobTargetPath, tt.sourcePath) require.Equal(t, tt.expectedTarget, result) }) @@ -531,7 +534,7 @@ func TestMoveWorker_ProcessWithResourceReferences(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Moving test/path1 to new/location/path1").Return() mockProgress.On("SetMessage", mock.Anything, "Moving dashboard/file.yaml to new/location/file.yaml").Return() mockProgress.On("TooManyErrors").Return(nil).Times(2) - + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(mockRepoResources, nil) mockRepoResources.On("FindResourcePath", mock.Anything, "dashboard-uid", schema.GroupVersionKind{ Group: "dashboard.grafana.app", @@ -555,7 +558,7 @@ func TestMoveWorker_ProcessWithResourceReferences(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) } @@ -594,6 +597,7 @@ func TestMoveWorker_ProcessResourceReferencesError(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Resolving resource paths").Return() mockProgress.On("SetMessage", mock.Anything, "Finding path for resource dashboard.grafana.app/Dashboard/non-existent-uid").Return() mockProgress.On("TooManyErrors").Return(nil) + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(mockRepoResources, nil) resourceError := errors.New("resource not found") @@ -616,7 +620,7 @@ func TestMoveWorker_ProcessResourceReferencesError(t *testing.T) { return syncJob.Spec.Pull != nil && !syncJob.Spec.Pull.Incremental }), mockProgress).Return(nil) - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) // Should continue despite individual resource errors } @@ -656,7 +660,7 @@ func TestMoveWorker_ProcessResourcesFactoryError(t *testing.T) { factoryError := errors.New("failed to create resources client") mockResourcesFactory.On("Client", mock.Anything, mockRepo).Return(nil, factoryError) - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.EqualError(t, err, "move files in repository: create repository resources client: failed to create resources client") } @@ -774,7 +778,7 @@ func TestMoveWorker_resolveResourcesToPaths(t *testing.T) { } } - worker := NewWorker(nil, nil, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, nil, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) paths, err := worker.resolveResourcesToPaths(context.Background(), mockRepo, mockProgress, tt.resources) if tt.expectedError != "" { @@ -863,7 +867,7 @@ func TestMoveWorker_RefURLsSetWithRef(t *testing.T) { mockProgress.On("Record", mock.Anything, mock.Anything).Once() mockProgress.On("TooManyErrors").Return(nil).Once() mockProgress.On("SetRefURLs", mock.Anything, expectedRefURLs).Once() - + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) mockReaderWriter := repository.NewMockReaderWriter(t) mockReaderWriter.On("Move", mock.Anything, "test.json", "target/test.json", "feature-branch", "Move test.json to target/test.json").Return(nil) @@ -886,7 +890,7 @@ func TestMoveWorker_RefURLsSetWithRef(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -916,6 +920,7 @@ func TestMoveWorker_RefURLsNotSetWithoutRef(t *testing.T) { mockProgress.On("TooManyErrors").Return(nil).Once() mockProgress.On("ResetResults").Once() mockProgress.On("SetMessage", mock.Anything, "pull resources").Once() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) // SetRefURLs should NOT be called since no ref is specified mockReaderWriter := repository.NewMockReaderWriter(t) @@ -943,7 +948,7 @@ func TestMoveWorker_RefURLsNotSetWithoutRef(t *testing.T) { }, } - worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(mockSyncWorker, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepoWithURLs, job, mockProgress) require.NoError(t, err) @@ -970,6 +975,7 @@ func TestMoveWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { mockProgress.On("SetMessage", mock.Anything, "Moving test.json to target/test.json").Once() mockProgress.On("Record", mock.Anything, mock.Anything).Once() mockProgress.On("TooManyErrors").Return(nil).Once() + mockProgress.On("Complete", mock.Anything, mock.Anything).Return(provisioning.JobStatus{}) // SetRefURLs should NOT be called since repo doesn't support URLs mockReaderWriter := repository.NewMockReaderWriter(t) @@ -994,7 +1000,7 @@ func TestMoveWorker_RefURLsNotSetForNonURLRepository(t *testing.T) { }, } - worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, prometheus.DefaultRegisterer) + worker := NewWorker(nil, mockWrapFn.Execute, mockResourcesFactory, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), mockRepo, job, mockProgress) require.NoError(t, err) diff --git a/pkg/registry/apis/provisioning/jobs/persistentstore.go b/pkg/registry/apis/provisioning/jobs/persistentstore.go index cd5b5c1561b..78494aa82cc 100644 --- a/pkg/registry/apis/provisioning/jobs/persistentstore.go +++ b/pkg/registry/apis/provisioning/jobs/persistentstore.go @@ -74,7 +74,7 @@ type persistentStore struct { // If a job is abandoned, it will have its claim cleaned up periodically. expiry time.Duration - registry prometheus.Registerer + queueMetrics QueueMetrics } // NewJobStore creates a new job queue implementation using the API client. @@ -83,11 +83,13 @@ func NewJobStore(provisioningClient client.ProvisioningV0alpha1Interface, expiry expiry = time.Second * 30 } + queueMetrics := RegisterQueueMetrics(registry) + return &persistentStore{ - client: provisioningClient, - clock: time.Now, - expiry: expiry, - registry: registry, + client: provisioningClient, + clock: time.Now, + expiry: expiry, + queueMetrics: queueMetrics, }, nil } @@ -120,6 +122,7 @@ func (s *persistentStore) Claim(ctx context.Context) (job *provisioning.Job, rol job.Labels = make(map[string]string) } job.Labels[LabelJobClaim] = strconv.FormatInt(s.clock().UnixMilli(), 10) + s.queueMetrics.RecordWaitTime(string(job.Spec.Action), s.clock().Sub(job.CreationTimestamp.Time).Seconds()) // Set up the provisioning identity for this namespace ctx, _, err = identity.WithProvisioningIdentity(ctx, job.GetNamespace()) @@ -241,6 +244,7 @@ func (s *persistentStore) Complete(ctx context.Context, job *provisioning.Job) e job.Labels = make(map[string]string) } delete(job.Labels, LabelJobClaim) + s.queueMetrics.DecreaseQueueSize(string(job.Spec.Action)) logger.Debug("job completion done") return nil @@ -383,6 +387,8 @@ func (s *persistentStore) Insert(ctx context.Context, namespace string, spec pro return nil, apifmt.Errorf("failed to create job '%s' in '%s': %w", job.GetName(), job.GetNamespace(), err) } + s.queueMetrics.IncreaseQueueSize(string(job.Spec.Action)) + return created, nil } diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker.go b/pkg/registry/apis/provisioning/jobs/sync/worker.go index a2e7f7feb69..0b06f1dbed3 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker.go @@ -3,6 +3,7 @@ package sync import ( "context" "fmt" + "time" "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" @@ -10,7 +11,6 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" - "github.com/prometheus/client_golang/prometheus" ) //go:generate mockery --name RepositoryPatchFn --structname MockRepositoryPatchFn --inpackage --filename repository_patch_fn_mock.go --with-expecter @@ -34,8 +34,7 @@ type SyncWorker struct { // Sync functions syncer Syncer - // Registry for metrics - registry prometheus.Registerer + metrics jobs.JobMetrics } func NewSyncWorker( @@ -44,7 +43,7 @@ func NewSyncWorker( storageStatus dualwrite.Service, patchStatus RepositoryPatchFn, syncer Syncer, - registry prometheus.Registerer, + metrics jobs.JobMetrics, ) *SyncWorker { return &SyncWorker{ clients: clients, @@ -52,7 +51,7 @@ func NewSyncWorker( patchStatus: patchStatus, storageStatus: storageStatus, syncer: syncer, - registry: registry, + metrics: metrics, } } @@ -63,6 +62,14 @@ func (r *SyncWorker) IsSupported(ctx context.Context, job provisioning.Job) bool func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, job provisioning.Job, progress jobs.JobProgressRecorder) error { cfg := repo.Config() logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) + + start := time.Now() + outcome := jobs.ErrorOutcome + totalChangesMade := 0 + defer func() { + r.metrics.RecordJob(string(provisioning.JobActionPull), outcome, totalChangesMade, time.Since(start).Seconds()) + }() + // Check if we are onboarding from legacy storage // HACK -- this should be handled outside of this worker if r.storageStatus != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, r.storageStatus) { @@ -71,7 +78,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo rw, ok := repo.(repository.ReaderWriter) if !ok { - return fmt.Errorf("sync job submitted for repository that does not support read-write -- this is a bug") + return fmt.Errorf("sync job submitted for repository that does not support read-write") } syncStatus := job.Status.ToSyncStatus(job.Name) @@ -90,16 +97,19 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo progress.SetMessage(ctx, "update sync status at start") if err := r.patchStatus(ctx, cfg, patchOperations...); err != nil { + logger.Error("failed to update the repository status at the start of the sync job", "error", err) return fmt.Errorf("update repo with job status at start: %w", err) } repositoryResources, err := r.repositoryResources.Client(ctx, rw) if err != nil { + logger.Error("failed to create repository resources client", "error", err) return fmt.Errorf("create repository resources client: %w", err) } clients, err := r.clients.Clients(ctx, cfg.Namespace) if err != nil { + logger.Error("failed to get clients for the repository", "error", err) return fmt.Errorf("get clients for %s: %w", cfg.Name, err) } @@ -110,6 +120,15 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo jobStatus := progress.Complete(ctx, syncError) syncStatus = jobStatus.ToSyncStatus(job.Name) + if syncError != nil { + logger.Debug("failed to sync the repository", "error", syncError) + } else { + outcome = jobs.SuccessOutcome + for _, summary := range jobStatus.Summary { + totalChangesMade += int(summary.Create + summary.Update + summary.Delete) + } + } + // Create sync status and set hash if successful if syncStatus.State == provisioning.JobStateSuccess { syncStatus.LastRef = currentRef @@ -146,6 +165,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo // Only patch the specific fields we want to update, not the entire status if err := r.patchStatus(ctx, cfg, patchOperations...); err != nil { + logger.Error("failed to update the repository status at the end of the sync job", "error", err) return fmt.Errorf("update repo with job final status: %w", err) } diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go index cce78138148..87ea058735d 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go @@ -17,6 +17,7 @@ import ( ) func TestSyncWorker_IsSupported(t *testing.T) { + metrics := jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()) tests := []struct { name string job provisioning.Job @@ -44,7 +45,7 @@ func TestSyncWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewSyncWorker(nil, nil, nil, nil, nil, prometheus.DefaultRegisterer) + worker := NewSyncWorker(nil, nil, nil, nil, nil, metrics) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -63,9 +64,9 @@ func TestSyncWorker_ProcessNotReaderWriter(t *testing.T) { }) fakeDualwrite := dualwrite.NewMockService(t) fakeDualwrite.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() - worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil, prometheus.DefaultRegisterer) + worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) err := worker.Process(context.Background(), repo, provisioning.Job{}, jobs.NewMockJobProgressRecorder(t)) - require.EqualError(t, err, "sync job submitted for repository that does not support read-write -- this is a bug") + require.EqualError(t, err, "sync job submitted for repository that does not support read-write") } func TestSyncWorker_Process(t *testing.T) { @@ -531,7 +532,7 @@ func TestSyncWorker_Process(t *testing.T) { dualwriteService, repositoryPatchFn.Execute, syncer, - prometheus.DefaultRegisterer, + jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()), ) // Create test job diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index 08a5c5cc034..461f3d7e27c 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -737,13 +737,15 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH usageMetricCollector := usage.MetricCollector(b.tracer, b.getRepositoriesInNamespace, b.unified) b.usageStats.RegisterMetricsFunc(usageMetricCollector) + metrics := jobs.RegisterJobMetrics(b.registry) + stageIfPossible := repository.WrapWithStageAndPushIfPossible exportWorker := export.NewExportWorker( b.clients, b.repositoryResources, export.ExportAll, stageIfPossible, - b.registry, + metrics, ) syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync) @@ -753,7 +755,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.storageStatus, b.statusPatcher.Patch, syncer, - b.registry, + metrics, ) signerFactory := signature.NewSignerFactory(b.clients) legacyResources := migrate.NewLegacyResourcesMigrator( @@ -785,8 +787,8 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.storageStatus, ) - deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources, b.registry) - moveWorker := movepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources, b.registry) + deleteWorker := deletepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources, metrics) + moveWorker := movepkg.NewWorker(syncWorker, stageIfPossible, b.repositoryResources, metrics) workers := []jobs.Worker{ deleteWorker, exportWorker, From a2d09490ae9c2dba4eb854e86832a56e59b20dc1 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Mon, 22 Sep 2025 14:07:22 -0600 Subject: [PATCH 22/54] Provisioning: Allow setting concurrent jobs settings and add metric (#111451) --- pkg/operators/provisioning/jobs_operator.go | 22 ++++++++++++++----- .../provisioning/jobs/concurrent_driver.go | 4 ++-- .../apis/provisioning/jobs/metrics.go | 12 ++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pkg/operators/provisioning/jobs_operator.go b/pkg/operators/provisioning/jobs_operator.go index e108e8a8d3d..462f281db3c 100644 --- a/pkg/operators/provisioning/jobs_operator.go +++ b/pkg/operators/provisioning/jobs_operator.go @@ -112,11 +112,11 @@ func RunJobController(deps server.OperatorDependencies) error { // This is basically our own JobQueue system driver, err := jobs.NewConcurrentJobDriver( - 3, // 3 drivers for now - 20*time.Minute, // Max time for each job - time.Minute, // Cleanup jobs - 30*time.Second, // Periodically look for new jobs - 30*time.Second, // Lease renewal interval + controllerCfg.concurrentDrivers, + controllerCfg.maxJobTimeout, + controllerCfg.cleanupInterval, + controllerCfg.jobInterval, + controllerCfg.leaseRenewalInterval, jobStore, repoGetter, jobHistoryWriter, @@ -150,7 +150,12 @@ func RunJobController(deps server.OperatorDependencies) error { type jobsControllerConfig struct { provisioningControllerConfig - historyExpiration time.Duration + historyExpiration time.Duration + maxJobTimeout time.Duration + cleanupInterval time.Duration + jobInterval time.Duration + leaseRenewalInterval time.Duration + concurrentDrivers int } func setupJobsControllerFromConfig(cfg *setting.Cfg, registry prometheus.Registerer) (*jobsControllerConfig, error) { @@ -162,6 +167,11 @@ func setupJobsControllerFromConfig(cfg *setting.Cfg, registry prometheus.Registe return &jobsControllerConfig{ provisioningControllerConfig: *controllerCfg, historyExpiration: cfg.SectionWithEnvOverrides("operator").Key("history_expiration").MustDuration(0), + concurrentDrivers: cfg.SectionWithEnvOverrides("operator").Key("concurrent_drivers").MustInt(3), + maxJobTimeout: cfg.SectionWithEnvOverrides("operator").Key("max_job_timeout").MustDuration(20 * time.Minute), + cleanupInterval: cfg.SectionWithEnvOverrides("operator").Key("cleanup_interval").MustDuration(time.Minute), + jobInterval: cfg.SectionWithEnvOverrides("operator").Key("job_interval").MustDuration(30 * time.Second), + leaseRenewalInterval: cfg.SectionWithEnvOverrides("operator").Key("lease_renewal_interval").MustDuration(30 * time.Second), }, nil } diff --git a/pkg/registry/apis/provisioning/jobs/concurrent_driver.go b/pkg/registry/apis/provisioning/jobs/concurrent_driver.go index 4da84e4ec84..1fdc32bea60 100644 --- a/pkg/registry/apis/provisioning/jobs/concurrent_driver.go +++ b/pkg/registry/apis/provisioning/jobs/concurrent_driver.go @@ -21,7 +21,6 @@ type ConcurrentJobDriver struct { repoGetter RepoGetter historicJobs HistoryWriter workers []Worker - registry prometheus.Registerer notifications chan struct{} } @@ -58,6 +57,8 @@ func NewConcurrentJobDriver( cleanupInterval = 5 * time.Minute // Maximum cleanup interval } + recordConcurrentDriverMetric(registry, numDrivers) + return &ConcurrentJobDriver{ numDrivers: numDrivers, jobTimeout: jobTimeout, @@ -69,7 +70,6 @@ func NewConcurrentJobDriver( historicJobs: historicJobs, workers: workers, notifications: notifications, - registry: registry, }, nil } diff --git a/pkg/registry/apis/provisioning/jobs/metrics.go b/pkg/registry/apis/provisioning/jobs/metrics.go index 8a4a8d20eb6..bde8bb303db 100644 --- a/pkg/registry/apis/provisioning/jobs/metrics.go +++ b/pkg/registry/apis/provisioning/jobs/metrics.go @@ -110,3 +110,15 @@ func getResourceCountBucket(count int) string { return "1000+" } } + +func recordConcurrentDriverMetric(registry prometheus.Registerer, numDrivers int) { + concurrentDriver := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "grafana_provisioning_jobs_concurrent_driver_num_drivers", + Help: "Number of concurrent job drivers", + }, + []string{}, + ) + registry.MustRegister(concurrentDriver) + concurrentDriver.WithLabelValues().Set(float64(numDrivers)) +} From 8b1caccc720e9dd6b70ece7e62a04e8d9fd7c0fd Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Mon, 22 Sep 2025 14:14:03 -0600 Subject: [PATCH 23/54] Provisioning: Add metrics for repo controller (#111450) --- apps/provisioning/go.mod | 2 +- apps/provisioning/pkg/controller/status.go | 9 +- .../pkg/controller/status_test.go | 5 +- pkg/operators/provisioning/jobs_operator.go | 2 +- pkg/operators/provisioning/repo_operator.go | 2 +- .../provisioning/controller/finalizers.go | 38 ++++++--- .../apis/provisioning/controller/health.go | 16 +++- .../provisioning/controller/health_test.go | 14 ++-- .../apis/provisioning/controller/metrics.go | 84 +++++++++++++++++++ .../provisioning/controller/repository.go | 3 + .../apis/provisioning/jobs/delete/worker.go | 5 +- .../apis/provisioning/jobs/export/worker.go | 5 +- .../apis/provisioning/jobs/metrics.go | 31 ++----- .../apis/provisioning/jobs/move/worker.go | 5 +- .../apis/provisioning/jobs/sync/worker.go | 5 +- pkg/registry/apis/provisioning/register.go | 2 +- .../apis/provisioning/utils/metrics.go | 25 ++++++ 17 files changed, 187 insertions(+), 66 deletions(-) create mode 100644 pkg/registry/apis/provisioning/controller/metrics.go create mode 100644 pkg/registry/apis/provisioning/utils/metrics.go diff --git a/apps/provisioning/go.mod b/apps/provisioning/go.mod index 732f69c3671..1ffeb70c4f3 100644 --- a/apps/provisioning/go.mod +++ b/apps/provisioning/go.mod @@ -11,7 +11,6 @@ require ( github.com/grafana/grafana/pkg/apimachinery v0.0.0-20250804150913-990f1c69ecc2 github.com/grafana/nanogit v0.0.0-20250723104447-68f58f5ecec0 github.com/migueleliasweb/go-github-mock v1.1.0 - github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 golang.org/x/oauth2 v0.30.0 k8s.io/apimachinery v0.34.1 @@ -54,6 +53,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect diff --git a/apps/provisioning/pkg/controller/status.go b/apps/provisioning/pkg/controller/status.go index ad63a169415..40ed29624c1 100644 --- a/apps/provisioning/pkg/controller/status.go +++ b/apps/provisioning/pkg/controller/status.go @@ -7,20 +7,17 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" - "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) type RepositoryStatusPatcher struct { - client client.ProvisioningV0alpha1Interface - registry prometheus.Registerer + client client.ProvisioningV0alpha1Interface } -func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface, registry prometheus.Registerer) *RepositoryStatusPatcher { +func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *RepositoryStatusPatcher { return &RepositoryStatusPatcher{ - client: client, - registry: registry, + client: client, } } diff --git a/apps/provisioning/pkg/controller/status_test.go b/apps/provisioning/pkg/controller/status_test.go index 12c21fb8c5b..77e7f737ea5 100644 --- a/apps/provisioning/pkg/controller/status_test.go +++ b/apps/provisioning/pkg/controller/status_test.go @@ -8,7 +8,6 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1/fake" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -17,7 +16,7 @@ import ( func TestNewRepositoryStatusPatcher(t *testing.T) { client := &fake.FakeProvisioningV0alpha1{} - patcher := NewRepositoryStatusPatcher(client, prometheus.DefaultRegisterer) + patcher := NewRepositoryStatusPatcher(client) require.NotNil(t, patcher) require.Equal(t, client, patcher.client) } @@ -104,7 +103,7 @@ func TestRepositoryStatusPatcher_Patch(t *testing.T) { client.AddReactor("patch", "repositories", tt.reactorFunc) } - patcher := NewRepositoryStatusPatcher(&client, prometheus.DefaultRegisterer) + patcher := NewRepositoryStatusPatcher(&client) err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations...) if tt.expectedError != "" { diff --git a/pkg/operators/provisioning/jobs_operator.go b/pkg/operators/provisioning/jobs_operator.go index 462f281db3c..fb9fd9c494e 100644 --- a/pkg/operators/provisioning/jobs_operator.go +++ b/pkg/operators/provisioning/jobs_operator.go @@ -180,7 +180,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis parsers := resources.NewParserFactory(clients) resourceLister := resources.NewResourceLister(controllerCfg.unified) repositoryResources := resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister) - statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), registry) + statusPatcher := controller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1()) workers := make([]jobs.Worker, 0) diff --git a/pkg/operators/provisioning/repo_operator.go b/pkg/operators/provisioning/repo_operator.go index b64cb991777..a15c2033d57 100644 --- a/pkg/operators/provisioning/repo_operator.go +++ b/pkg/operators/provisioning/repo_operator.go @@ -55,7 +55,7 @@ func RunRepoController(deps server.OperatorDependencies) error { if err != nil { return fmt.Errorf("create API client job store: %w", err) } - statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1(), deps.Registerer) + statusPatcher := appcontroller.NewRepositoryStatusPatcher(controllerCfg.provisioningClient.ProvisioningV0alpha1()) healthChecker := controller.NewHealthChecker(statusPatcher, deps.Registerer) repoInformer := informerFactory.Provisioning().V0alpha1().Repositories() diff --git a/pkg/registry/apis/provisioning/controller/finalizers.go b/pkg/registry/apis/provisioning/controller/finalizers.go index 72c2209911c..952d11bf0f8 100644 --- a/pkg/registry/apis/provisioning/controller/finalizers.go +++ b/pkg/registry/apis/provisioning/controller/finalizers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "sort" "strings" + "time" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -17,11 +18,13 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + metricutils "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" ) type finalizer struct { lister resources.ResourceLister clientFactory resources.ClientFactory + metrics *finalizerMetrics } func (f *finalizer) process(ctx context.Context, @@ -31,18 +34,24 @@ func (f *finalizer) process(ctx context.Context, logger := logging.FromContext(ctx) for _, finalizer := range finalizers { + var err error + var count int + start := time.Now() + outcome := metricutils.SuccessOutcome + switch finalizer { case repository.CleanFinalizer: // NOTE: the controller loop will never get run unless a finalizer is set hooks, ok := repo.(repository.Hooks) if ok { - if err := hooks.OnDelete(ctx); err != nil { + if err = hooks.OnDelete(ctx); err != nil { logger.Warn("Error running deletion hooks", "err", err) + outcome = metricutils.ErrorOutcome } } case repository.ReleaseOrphanResourcesFinalizer: - err := f.processExistingItems(ctx, repo.Config(), + count, err = f.processExistingItems(ctx, repo.Config(), func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error { patchAnnotations, err := getPatchedAnnotations(item) if err != nil { @@ -55,20 +64,29 @@ func (f *finalizer) process(ctx context.Context, return err }) if err != nil { - return err + outcome = metricutils.ErrorOutcome + logger.Warn("Error processing release orphan resources finalizer", "err", err) } case repository.RemoveOrphanResourcesFinalizer: - err := f.processExistingItems(ctx, repo.Config(), + count, err = f.processExistingItems(ctx, repo.Config(), func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error { return client.Delete(ctx, item.Name, v1.DeleteOptions{}) }) if err != nil { - return err + outcome = metricutils.ErrorOutcome + logger.Warn("Error processing remove orphan resources finalizer", "err", err) } default: logger.Warn("skipping unknown finalizer", "finalizer", finalizer) + continue + } + + f.metrics.RecordFinalizer(finalizer, outcome, count, time.Since(start).Seconds()) + + if err != nil { + return err } } return nil @@ -79,17 +97,17 @@ func (f *finalizer) processExistingItems( ctx context.Context, repo *provisioning.Repository, cb func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error, -) error { +) (int, error) { logger := logging.FromContext(ctx) clients, err := f.clientFactory.Clients(ctx, repo.Namespace) if err != nil { - return err + return 0, err } items, err := f.lister.List(ctx, repo.Namespace, repo.Name) if err != nil { logger.Warn("error listing resources", "error", err) - return err + return 0, err } // Safe deletion order @@ -103,7 +121,7 @@ func (f *finalizer) processExistingItems( Resource: item.Resource, }) if err != nil { - return err + return count, err } err = cb(res, &item) @@ -115,7 +133,7 @@ func (f *finalizer) processExistingItems( } } logger.Info("processed orphan items", "items", count, "errors", errors) - return nil + return count, nil } type jsonPatchOperation struct { diff --git a/pkg/registry/apis/provisioning/controller/health.go b/pkg/registry/apis/provisioning/controller/health.go index 0d24e457274..cf3aeedc36d 100644 --- a/pkg/registry/apis/provisioning/controller/health.go +++ b/pkg/registry/apis/provisioning/controller/health.go @@ -5,8 +5,10 @@ import ( "fmt" "time" + "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" "github.com/prometheus/client_golang/prometheus" ) @@ -20,14 +22,15 @@ type StatusPatcher interface { // HealthChecker provides unified health checking for repositories type HealthChecker struct { statusPatcher StatusPatcher - registry prometheus.Registerer + healthMetrics healthMetrics } // NewHealthChecker creates a new health checker func NewHealthChecker(statusPatcher StatusPatcher, registry prometheus.Registerer) *HealthChecker { + healthMetrics := registerHealthMetrics(registry) return &HealthChecker{ statusPatcher: statusPatcher, - registry: registry, + healthMetrics: healthMetrics, } } @@ -166,8 +169,17 @@ func (hc *HealthChecker) RefreshTimestamp(ctx context.Context, repo *provisionin // refreshHealth performs a comprehensive health check // Returns test results, health status, and any error func (hc *HealthChecker) refreshHealth(ctx context.Context, repo repository.Repository, existingStatus provisioning.HealthStatus) (*provisioning.TestResults, provisioning.HealthStatus, error) { + logger := logging.FromContext(ctx).With("repo", repo.Config().GetName(), "namespace", repo.Config().GetNamespace()) + start := time.Now() + outcome := utils.SuccessOutcome + defer func() { + hc.healthMetrics.RecordHealthCheck(outcome, time.Since(start).Seconds()) + }() + res, err := repository.TestRepository(ctx, repo) if err != nil { + outcome = utils.ErrorOutcome + logger.Error("failed to test repository", "error", err) return nil, existingStatus, fmt.Errorf("failed to test repository: %w", err) } diff --git a/pkg/registry/apis/provisioning/controller/health_test.go b/pkg/registry/apis/provisioning/controller/health_test.go index c9081875b2a..74f8a127be6 100644 --- a/pkg/registry/apis/provisioning/controller/health_test.go +++ b/pkg/registry/apis/provisioning/controller/health_test.go @@ -19,7 +19,7 @@ import ( func TestNewHealthChecker(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) assert.NotNil(t, hc) assert.Equal(t, mockPatcher, hc.statusPatcher) @@ -136,7 +136,7 @@ func TestShouldCheckHealth(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) result := hc.ShouldCheckHealth(tt.repo) assert.Equal(t, tt.expected, result) @@ -223,7 +223,7 @@ func TestHasRecentFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) result := hc.HasRecentFailure(tt.healthStatus, tt.failureType) assert.Equal(t, tt.expected, result) @@ -265,7 +265,7 @@ func TestRecordFailure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) repo := &provisioning.Repository{ Status: provisioning.RepositoryStatus{ @@ -310,7 +310,7 @@ func TestRecordFailure(t *testing.T) { func TestRecordFailureFunction(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) testErr := errors.New("test error") result := hc.recordFailure(provisioning.HealthFailureHook, testErr) @@ -447,7 +447,7 @@ func TestRefreshHealth(t *testing.T) { testError: tt.testError, } - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) if tt.expectPatch { if tt.patchError != nil { @@ -557,7 +557,7 @@ func TestHasHealthStatusChanged(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockPatcher := mocks.NewStatusPatcher(t) - hc := NewHealthChecker(mockPatcher, prometheus.DefaultRegisterer) + hc := NewHealthChecker(mockPatcher, prometheus.NewPedanticRegistry()) result := hc.hasHealthStatusChanged(tt.old, tt.new) assert.Equal(t, tt.expected, result) diff --git a/pkg/registry/apis/provisioning/controller/metrics.go b/pkg/registry/apis/provisioning/controller/metrics.go new file mode 100644 index 00000000000..a6850b438c9 --- /dev/null +++ b/pkg/registry/apis/provisioning/controller/metrics.go @@ -0,0 +1,84 @@ +package controller + +import ( + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" + "github.com/prometheus/client_golang/prometheus" +) + +type finalizerMetrics struct { + registry prometheus.Registerer + finalizerProcessedTotal *prometheus.CounterVec + finalizerDuration *prometheus.HistogramVec +} + +func registerFinalizerMetrics(registry prometheus.Registerer) finalizerMetrics { + finalizerProcessedTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "grafana_provisioning_finalizers_processed_total", + Help: "Total number of finalizers processed", + }, + []string{"finalizer_type", "outcome"}, + ) + registry.MustRegister(finalizerProcessedTotal) + + finalizerDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grafana_provisioning_finalizers_duration_seconds", + Help: "Duration of processing the finalizers", + Buckets: []float64{0.5, 1.0, 2.0, 5.0, 10.0, 30.0}, + }, + []string{"finalizer_type", "resource_count_bucket"}, + ) + registry.MustRegister(finalizerDuration) + + return finalizerMetrics{ + registry: registry, + finalizerProcessedTotal: finalizerProcessedTotal, + finalizerDuration: finalizerDuration, + } +} + +func (m *finalizerMetrics) RecordFinalizer(finalizerType string, outcome string, resourceCountChanged int, duration float64) { + m.finalizerProcessedTotal.WithLabelValues(finalizerType, outcome).Inc() + if outcome == utils.SuccessOutcome { + m.finalizerDuration.WithLabelValues(finalizerType, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration) + } +} + +type healthMetrics struct { + registry prometheus.Registerer + healthCheckedTotal *prometheus.CounterVec + healthCheckedDuration *prometheus.HistogramVec +} + +func registerHealthMetrics(registry prometheus.Registerer) healthMetrics { + healthCheckedTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "grafana_provisioning_health_checked_total", + Help: "Total number of health checks performed", + }, + []string{"outcome"}, + ) + registry.MustRegister(healthCheckedTotal) + + healthCheckedDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grafana_provisioning_health_checked_duration_seconds", + Help: "Duration of health checks", + Buckets: []float64{0.1, 0.2, 0.5, 1.0, 2.0, 5.0}, + }, + []string{}, + ) + registry.MustRegister(healthCheckedDuration) + + return healthMetrics{ + registry: registry, + healthCheckedTotal: healthCheckedTotal, + healthCheckedDuration: healthCheckedDuration, + } +} + +func (m *healthMetrics) RecordHealthCheck(outcome string, duration float64) { + m.healthCheckedTotal.WithLabelValues(outcome).Inc() + m.healthCheckedDuration.WithLabelValues().Observe(duration) +} diff --git a/pkg/registry/apis/provisioning/controller/repository.go b/pkg/registry/apis/provisioning/controller/repository.go index 9ddaf4fa476..cbc55781819 100644 --- a/pkg/registry/apis/provisioning/controller/repository.go +++ b/pkg/registry/apis/provisioning/controller/repository.go @@ -77,6 +77,8 @@ func NewRepositoryController( statusPatcher StatusPatcher, registry prometheus.Registerer, ) (*RepositoryController, error) { + finalizerMetrics := registerFinalizerMetrics(registry) + rc := &RepositoryController{ client: provisioningClient, repoLister: repoInformer.Lister(), @@ -93,6 +95,7 @@ func NewRepositoryController( finalizer: &finalizer{ lister: resourceLister, clientFactory: clients, + metrics: &finalizerMetrics, }, jobs: jobs, logger: logging.DefaultLogger.With("logger", loggerName), diff --git a/pkg/registry/apis/provisioning/jobs/delete/worker.go b/pkg/registry/apis/provisioning/jobs/delete/worker.go index dd627f4184c..e06ae627917 100644 --- a/pkg/registry/apis/provisioning/jobs/delete/worker.go +++ b/pkg/registry/apis/provisioning/jobs/delete/worker.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" ) type Worker struct { @@ -44,7 +45,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr opts := *job.Spec.Delete paths := opts.Paths start := time.Now() - outcome := jobs.ErrorOutcome + outcome := utils.ErrorOutcome resourcesDeleted := 0 defer func() { w.metrics.RecordJob(string(provisioning.JobActionDelete), outcome, resourcesDeleted, time.Since(start).Seconds()) @@ -119,7 +120,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr } } - outcome = jobs.SuccessOutcome + outcome = utils.SuccessOutcome jobStatus := progress.Complete(ctx, nil) for _, summary := range jobStatus.Summary { resourcesDeleted += int(summary.Delete) diff --git a/pkg/registry/apis/provisioning/jobs/export/worker.go b/pkg/registry/apis/provisioning/jobs/export/worker.go index 8835a7b6faa..092243c4567 100644 --- a/pkg/registry/apis/provisioning/jobs/export/worker.go +++ b/pkg/registry/apis/provisioning/jobs/export/worker.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" ) //go:generate mockery --name ExportFn --structname MockExportFn --inpackage --filename mock_export_fn.go --with-expecter @@ -56,7 +57,7 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) start := time.Now() - outcome := jobs.ErrorOutcome + outcome := utils.ErrorOutcome resourcesExported := 0 defer func() { r.metrics.RecordJob(string(provisioning.JobActionPush), outcome, resourcesExported, time.Since(start).Seconds()) @@ -118,7 +119,7 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository, return err } - outcome = jobs.SuccessOutcome + outcome = utils.SuccessOutcome jobStatus := progress.Complete(ctx, nil) for _, summary := range jobStatus.Summary { resourcesExported += int(summary.Write) diff --git a/pkg/registry/apis/provisioning/jobs/metrics.go b/pkg/registry/apis/provisioning/jobs/metrics.go index bde8bb303db..dc946838659 100644 --- a/pkg/registry/apis/provisioning/jobs/metrics.go +++ b/pkg/registry/apis/provisioning/jobs/metrics.go @@ -1,10 +1,8 @@ package jobs -import "github.com/prometheus/client_golang/prometheus" - -const ( - SuccessOutcome = "success" - ErrorOutcome = "error" +import ( + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" + "github.com/prometheus/client_golang/prometheus" ) type JobMetrics struct { @@ -87,27 +85,8 @@ func (m *JobMetrics) RecordJob(jobAction string, outcome string, resourceCountCh m.processedTotal.WithLabelValues(jobAction, outcome).Inc() // only record duration when the job was successful. otherwise resource count will be incorrect - if outcome == SuccessOutcome { - m.durationHist.WithLabelValues(jobAction, getResourceCountBucket(resourceCountChanged)).Observe(duration) - } -} - -func getResourceCountBucket(count int) string { - switch { - case count == 0: - return "0" - case count <= 10: - return "1-10" - case count <= 50: - return "11-50" - case count <= 100: - return "51-100" - case count <= 500: - return "101-500" - case count <= 1000: - return "501-1000" - default: - return "1000+" + if outcome == utils.SuccessOutcome { + m.durationHist.WithLabelValues(jobAction, utils.GetResourceCountBucket(resourceCountChanged)).Observe(duration) } } diff --git a/pkg/registry/apis/provisioning/jobs/move/worker.go b/pkg/registry/apis/provisioning/jobs/move/worker.go index 6123bad5278..1ec3d817b3e 100644 --- a/pkg/registry/apis/provisioning/jobs/move/worker.go +++ b/pkg/registry/apis/provisioning/jobs/move/worker.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/safepath" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" ) type Worker struct { @@ -43,7 +44,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr } opts := *job.Spec.Move logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) - outcome := jobs.ErrorOutcome + outcome := utils.ErrorOutcome start := time.Now() resourcesMoved := 0 defer func() { @@ -129,7 +130,7 @@ func (w *Worker) Process(ctx context.Context, repo repository.Repository, job pr } } - outcome = jobs.SuccessOutcome + outcome = utils.SuccessOutcome jobStatus := progress.Complete(ctx, nil) for _, summary := range jobStatus.Summary { // FileActionRenamed increments both delete & create, use create here diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker.go b/pkg/registry/apis/provisioning/jobs/sync/worker.go index 0b06f1dbed3..5443c814bcf 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" ) @@ -64,7 +65,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) start := time.Now() - outcome := jobs.ErrorOutcome + outcome := utils.ErrorOutcome totalChangesMade := 0 defer func() { r.metrics.RecordJob(string(provisioning.JobActionPull), outcome, totalChangesMade, time.Since(start).Seconds()) @@ -123,7 +124,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo if syncError != nil { logger.Debug("failed to sync the repository", "error", syncError) } else { - outcome = jobs.SuccessOutcome + outcome = utils.SuccessOutcome for _, summary := range jobStatus.Summary { totalChangesMade += int(summary.Create + summary.Update + summary.Delete) } diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index 461f3d7e27c..1904c417071 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -716,7 +716,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH return fmt.Errorf("create API client job store: %w", err) } - b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient(), b.registry) + b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient()) b.healthChecker = controller.NewHealthChecker(b.statusPatcher, b.registry) // if running solely CRUD, skip the rest of the setup diff --git a/pkg/registry/apis/provisioning/utils/metrics.go b/pkg/registry/apis/provisioning/utils/metrics.go new file mode 100644 index 00000000000..95171aed588 --- /dev/null +++ b/pkg/registry/apis/provisioning/utils/metrics.go @@ -0,0 +1,25 @@ +package utils + +const ( + SuccessOutcome = "success" + ErrorOutcome = "error" +) + +func GetResourceCountBucket(count int) string { + switch { + case count == 0: + return "0" + case count <= 10: + return "1-10" + case count <= 50: + return "11-50" + case count <= 100: + return "51-100" + case count <= 500: + return "101-500" + case count <= 1000: + return "501-1000" + default: + return "1000+" + } +} From 63e1d52663812f4663c50d65dd81252a7d939c98 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Mon, 22 Sep 2025 15:53:50 -0600 Subject: [PATCH 24/54] Provisioning: Add metrics around webhooks (#111453) --- .../apis/provisioning/webhooks/metrics.go | 30 ++++++ .../webhooks/pullrequest/changes.go | 19 +++- .../webhooks/pullrequest/changes_test.go | 6 +- .../webhooks/pullrequest/metrics.go | 97 +++++++++++++++++++ .../webhooks/pullrequest/worker.go | 32 +++++- .../webhooks/pullrequest/worker_test.go | 12 ++- .../apis/provisioning/webhooks/register.go | 4 +- .../apis/provisioning/webhooks/webhook.go | 10 ++ pkg/server/wire_gen.go | 4 +- 9 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 pkg/registry/apis/provisioning/webhooks/metrics.go create mode 100644 pkg/registry/apis/provisioning/webhooks/pullrequest/metrics.go diff --git a/pkg/registry/apis/provisioning/webhooks/metrics.go b/pkg/registry/apis/provisioning/webhooks/metrics.go new file mode 100644 index 00000000000..1323c12233d --- /dev/null +++ b/pkg/registry/apis/provisioning/webhooks/metrics.go @@ -0,0 +1,30 @@ +package webhooks + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type webhookMetrics struct { + registry prometheus.Registerer + eventsProcessed *prometheus.CounterVec +} + +func registerWebhookMetrics(registry prometheus.Registerer) webhookMetrics { + eventsProcessed := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "grafana_provisioning_webhook_events_processed_total", + Help: "Total number of webhook events processed and what job type was queued", + }, + []string{"job"}, + ) + registry.MustRegister(eventsProcessed) + + return webhookMetrics{ + registry: registry, + eventsProcessed: eventsProcessed, + } +} + +func (m *webhookMetrics) recordEventProcessed(jobQueued string) { + m.eventsProcessed.WithLabelValues(jobQueued).Inc() +} diff --git a/pkg/registry/apis/provisioning/webhooks/pullrequest/changes.go b/pkg/registry/apis/provisioning/webhooks/pullrequest/changes.go index 6f50fbfba31..4c9a49a7db4 100644 --- a/pkg/registry/apis/provisioning/webhooks/pullrequest/changes.go +++ b/pkg/registry/apis/provisioning/webhooks/pullrequest/changes.go @@ -6,6 +6,7 @@ import ( "net/url" "path" "strings" + "time" "github.com/grafana/grafana-app-sdk/logging" dashboard "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v1beta1" @@ -15,6 +16,8 @@ import ( "github.com/grafana/grafana/pkg/infra/slugify" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" + "github.com/prometheus/client_golang/prometheus" ) type changeInfo struct { @@ -53,13 +56,16 @@ type evaluator struct { render ScreenshotRenderer parsers resources.ParserFactory urlProvider func(namespace string) string + metrics screenshotMetrics } -func NewEvaluator(render ScreenshotRenderer, parsers resources.ParserFactory, urlProvider func(namespace string) string) Evaluator { +func NewEvaluator(render ScreenshotRenderer, parsers resources.ParserFactory, urlProvider func(namespace string) string, registry prometheus.Registerer) Evaluator { + metrics := registerScreenshotMetrics(registry) return &evaluator{ render: render, parsers: parsers, urlProvider: urlProvider, + metrics: metrics, } } @@ -152,14 +158,14 @@ func (e *evaluator) evaluateFile(ctx context.Context, repo repository.Reader, ba info.PreviewURL += "?" + query.Encode() if shouldRender { if info.GrafanaURL != "" { - info.GrafanaScreenshotURL, err = renderScreenshotFromGrafanaURL(ctx, baseURL, e.render, info.Parsed.Repo, info.GrafanaURL) + info.GrafanaScreenshotURL, err = renderScreenshotFromGrafanaURL(ctx, baseURL, e.render, info.Parsed.Repo, info.GrafanaURL, e.metrics) if err != nil { info.Error = err.Error() } } if info.PreviewURL != "" { - info.PreviewScreenshotURL, err = renderScreenshotFromGrafanaURL(ctx, baseURL, e.render, info.Parsed.Repo, info.PreviewURL) + info.PreviewScreenshotURL, err = renderScreenshotFromGrafanaURL(ctx, baseURL, e.render, info.Parsed.Repo, info.PreviewURL, e.metrics) if err != nil { info.Error = err.Error() } @@ -175,7 +181,13 @@ func renderScreenshotFromGrafanaURL(ctx context.Context, renderer ScreenshotRenderer, repo provisioning.ResourceRepositoryInfo, grafanaURL string, + metrics screenshotMetrics, ) (string, error) { + outcome := utils.ErrorOutcome + duration := time.Now() + defer func() { + metrics.recordScreenshotDuration(outcome, time.Since(duration)) + }() parsed, err := url.Parse(grafanaURL) if err != nil { logging.FromContext(ctx).Warn("invalid", "url", grafanaURL, "err", err) @@ -194,5 +206,6 @@ func renderScreenshotFromGrafanaURL(ctx context.Context, logger.Warn("invalid base", "url", baseURL, "err", err) return "", err } + outcome = utils.SuccessOutcome return base.JoinPath(snap).String(), nil } diff --git a/pkg/registry/apis/provisioning/webhooks/pullrequest/changes_test.go b/pkg/registry/apis/provisioning/webhooks/pullrequest/changes_test.go index fef0cebfc16..1d647256a2c 100644 --- a/pkg/registry/apis/provisioning/webhooks/pullrequest/changes_test.go +++ b/pkg/registry/apis/provisioning/webhooks/pullrequest/changes_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -759,7 +760,7 @@ func TestCalculateChanges(t *testing.T) { } return "http://host/" - }) + }, prometheus.NewPedanticRegistry()) pullRequest := provisioning.PullRequestJobOptions{ Ref: "ref", @@ -890,6 +891,7 @@ func TestRenderScreenshotFromGrafanaURL(t *testing.T) { }, } + metrics := registerScreenshotMetrics(prometheus.NewPedanticRegistry()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { renderer := NewMockScreenshotRenderer(t) @@ -900,7 +902,7 @@ func TestRenderScreenshotFromGrafanaURL(t *testing.T) { Name: "repo", } - got, err := renderScreenshotFromGrafanaURL(context.Background(), tt.baseURL, renderer, repo, tt.grafanaURL) + got, err := renderScreenshotFromGrafanaURL(context.Background(), tt.baseURL, renderer, repo, tt.grafanaURL, metrics) if tt.wantErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErr) diff --git a/pkg/registry/apis/provisioning/webhooks/pullrequest/metrics.go b/pkg/registry/apis/provisioning/webhooks/pullrequest/metrics.go new file mode 100644 index 00000000000..651f07795bb --- /dev/null +++ b/pkg/registry/apis/provisioning/webhooks/pullrequest/metrics.go @@ -0,0 +1,97 @@ +package pullrequest + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + pullRequestMetricsOnce sync.Once + pullRequestMetricsInstance pullRequestMetrics +) + +type pullRequestMetrics struct { + registry prometheus.Registerer + processingDuration *prometheus.HistogramVec + commentsPosted *prometheus.CounterVec +} + +func registerPullRequestMetrics(registry prometheus.Registerer) pullRequestMetrics { + // called by both ProvidePullRequestWorker and ProvideWebhooksWithImages + // this ensures we only register the metric once + pullRequestMetricsOnce.Do(func() { + processingDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grafana_provisioning_pullrequest_processing_duration_seconds", + Help: "Duration of pull request processing", + Buckets: []float64{0.5, 1.0, 2.0, 5.0, 10.0, 30.0}, + }, + []string{"outcome"}, + ) + registry.MustRegister(processingDuration) + + commentsPosted := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "grafana_provisioning_pullrequest_comments_posted_total", + Help: "Total number of comments posted to pull requests", + }, + []string{"outcome"}, + ) + registry.MustRegister(commentsPosted) + + pullRequestMetricsInstance = pullRequestMetrics{ + registry: registry, + processingDuration: processingDuration, + commentsPosted: commentsPosted, + } + }) + + return pullRequestMetricsInstance +} + +func (m *pullRequestMetrics) recordProcessed(outcome string, duration time.Duration) { + m.processingDuration.WithLabelValues(outcome).Observe(duration.Seconds()) +} + +func (m *pullRequestMetrics) recordCommentPosted(outcome string) { + m.commentsPosted.WithLabelValues(outcome).Inc() +} + +type screenshotMetrics struct { + registry prometheus.Registerer + screenshotDuration *prometheus.HistogramVec +} + +var ( + screenshotMetricsOnce sync.Once + screenshotMetricsInstance screenshotMetrics +) + +func registerScreenshotMetrics(registry prometheus.Registerer) screenshotMetrics { + // called by both ProvidePullRequestWorker and ProvideWebhooksWithImages + // this ensures we only register the metric once + screenshotMetricsOnce.Do(func() { + screenshotDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "grafana_provisioning_pullrequest_screenshot_duration_seconds", + Help: "Duration of screenshot generation", + Buckets: []float64{1.0, 2.0, 5.0, 10.0, 30.0, 60.0}, + }, + []string{"outcome"}, + ) + registry.MustRegister(screenshotDuration) + + screenshotMetricsInstance = screenshotMetrics{ + registry: registry, + screenshotDuration: screenshotDuration, + } + }) + + return screenshotMetricsInstance +} + +func (m *screenshotMetrics) recordScreenshotDuration(outcome string, duration time.Duration) { + m.screenshotDuration.WithLabelValues(outcome).Observe(duration.Seconds()) +} diff --git a/pkg/registry/apis/provisioning/webhooks/pullrequest/worker.go b/pkg/registry/apis/provisioning/webhooks/pullrequest/worker.go index 1291c590148..299febe9786 100644 --- a/pkg/registry/apis/provisioning/webhooks/pullrequest/worker.go +++ b/pkg/registry/apis/provisioning/webhooks/pullrequest/worker.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -12,10 +13,12 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" "github.com/grafana/grafana/pkg/services/apiserver" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" + "github.com/prometheus/client_golang/prometheus" ) func ProvidePullRequestWorker( @@ -23,6 +26,7 @@ func ProvidePullRequestWorker( renderer rendering.Service, blobstore resource.ResourceClient, configProvider apiserver.RestConfigProvider, + registry prometheus.Registerer, ) *PullRequestWorker { urlProvider := func(_ string) string { return cfg.AppURL @@ -33,10 +37,10 @@ func ProvidePullRequestWorker( clients := resources.NewClientFactory(configProvider) parsers := resources.NewParserFactory(clients) screenshotRenderer := NewScreenshotRenderer(renderer, blobstore) - evaluator := NewEvaluator(screenshotRenderer, parsers, urlProvider) + evaluator := NewEvaluator(screenshotRenderer, parsers, urlProvider, registry) commenter := NewCommenter() - return NewPullRequestWorker(evaluator, commenter) + return NewPullRequestWorker(evaluator, commenter, registry) } //go:generate mockery --name=PullRequestRepo --structname=MockPullRequestRepo --inpackage --filename=mock_pullrequest_repo.go --with-expecter @@ -60,12 +64,15 @@ type Commenter interface { type PullRequestWorker struct { evaluator Evaluator commenter Commenter + metrics pullRequestMetrics } -func NewPullRequestWorker(evaluator Evaluator, commenter Commenter) *PullRequestWorker { +func NewPullRequestWorker(evaluator Evaluator, commenter Commenter, registry prometheus.Registerer) *PullRequestWorker { + metrics := registerPullRequestMetrics(registry) return &PullRequestWorker{ evaluator: evaluator, commenter: commenter, + metrics: metrics, } } @@ -80,30 +87,42 @@ func (c *PullRequestWorker) Process(ctx context.Context, ) error { cfg := repo.Config().Spec opts := job.Spec.PullRequest + startTime := time.Now() + outcome := utils.ErrorOutcome + defer func() { + duration := time.Since(startTime) + c.metrics.recordProcessed(outcome, duration) + }() + if opts == nil { return apierrors.NewBadRequest("missing spec.pr") } + logger := logging.FromContext(ctx).With("pr", opts.PR, "repo", repo.Config().GetName(), "namespace", job.GetNamespace()) + if opts.Ref == "" { + logger.Debug("missing spec.ref") return apierrors.NewBadRequest("missing spec.ref") } // FIXME: this is leaky because it's supposed to be already a PullRequestRepo if cfg.GitHub == nil { + logger.Debug("expecting github configuration") return apierrors.NewBadRequest("expecting github configuration") } reader, ok := repo.(repository.Reader) if !ok { + logger.Debug("pull request job submitted targeting repository that is not a Reader") return errors.New("pull request job submitted targeting repository that is not a Reader") } prRepo, ok := repo.(PullRequestRepo) if !ok { + logger.Debug("pull request job submitted targeting repository that is not a PullRequestRepo") return fmt.Errorf("repository is not a pull request repository") } - logger := logging.FromContext(ctx).With("pr", opts.PR) logger.Info("process pull request") defer logger.Info("pull request processed") @@ -112,6 +131,7 @@ func (c *PullRequestWorker) Process(ctx context.Context, base := cfg.GitHub.Branch files, err := prRepo.CompareFiles(ctx, base, opts.Ref) if err != nil { + logger.Error("failed to list pull request files", "error", err) return fmt.Errorf("failed to list pull request files: %w", err) } @@ -123,12 +143,16 @@ func (c *PullRequestWorker) Process(ctx context.Context, changeInfo, err := c.evaluator.Evaluate(ctx, reader, *opts, files, progress) if err != nil { + logger.Error("failed to calculate changes", "error", err) return fmt.Errorf("calculate changes: %w", err) } if err := c.commenter.Comment(ctx, prRepo, opts.PR, changeInfo); err != nil { + c.metrics.recordCommentPosted(utils.ErrorOutcome) return fmt.Errorf("comment pull request: %w", err) } + outcome = utils.SuccessOutcome + c.metrics.recordCommentPosted(utils.SuccessOutcome) logger.Info("preview comment added") return nil diff --git a/pkg/registry/apis/provisioning/webhooks/pullrequest/worker_test.go b/pkg/registry/apis/provisioning/webhooks/pullrequest/worker_test.go index 7f1cb4f14a0..3414703a5bf 100644 --- a/pkg/registry/apis/provisioning/webhooks/pullrequest/worker_test.go +++ b/pkg/registry/apis/provisioning/webhooks/pullrequest/worker_test.go @@ -5,10 +5,12 @@ import ( "errors" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" @@ -44,7 +46,7 @@ func TestPullRequestWorker_IsSupported(t *testing.T) { t.Run(tt.name, func(t *testing.T) { evaluator := NewMockEvaluator(t) commenter := NewMockCommenter(t) - worker := NewPullRequestWorker(evaluator, commenter) + worker := NewPullRequestWorker(evaluator, commenter, prometheus.NewPedanticRegistry()) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -68,7 +70,7 @@ func TestPullRequestWorker_Process_NotPullRequestRepository(t *testing.T) { }, }) - worker := NewPullRequestWorker(evaluator, commenter) + worker := NewPullRequestWorker(evaluator, commenter, prometheus.NewPedanticRegistry()) job := provisioning.Job{ Spec: provisioning.JobSpec{ Action: provisioning.JobActionPullRequest, @@ -106,7 +108,7 @@ func TestPullRequestWorker_Process_NotReaderRepository(t *testing.T) { }, }) - worker := NewPullRequestWorker(evaluator, commenter) + worker := NewPullRequestWorker(evaluator, commenter, prometheus.NewPedanticRegistry()) job := provisioning.Job{ Spec: provisioning.JobSpec{ Action: provisioning.JobActionPullRequest, @@ -392,7 +394,7 @@ func TestPullRequestWorker_Process(t *testing.T) { progress := jobs.NewMockJobProgressRecorder(t) tt.setupMocks(evaluator, commenter, &repo, progress) - worker := NewPullRequestWorker(evaluator, commenter) + worker := NewPullRequestWorker(evaluator, commenter, prometheus.NewPedanticRegistry()) job := provisioning.Job{ Spec: provisioning.JobSpec{ Action: provisioning.JobActionPullRequest, @@ -400,7 +402,7 @@ func TestPullRequestWorker_Process(t *testing.T) { }, } - err := worker.Process(context.Background(), repo, job, progress) + err := worker.Process(logging.Context(context.Background(), logging.DefaultLogger), repo, job, progress) if tt.expectedError != "" { require.EqualError(t, err, tt.expectedError) } else { diff --git a/pkg/registry/apis/provisioning/webhooks/register.go b/pkg/registry/apis/provisioning/webhooks/register.go index 6dc55a5b0de..4fc02db9041 100644 --- a/pkg/registry/apis/provisioning/webhooks/register.go +++ b/pkg/registry/apis/provisioning/webhooks/register.go @@ -87,9 +87,9 @@ func ProvideWebhooksWithImages( registry, ) - evaluator := pullrequest.NewEvaluator(screenshotRenderer, parsers, urlProvider) + evaluator := pullrequest.NewEvaluator(screenshotRenderer, parsers, urlProvider, registry) commenter := pullrequest.NewCommenter() - pullRequestWorker := pullrequest.NewPullRequestWorker(evaluator, commenter) + pullRequestWorker := pullrequest.NewPullRequestWorker(evaluator, commenter, registry) return NewWebhookExtraWithImages( render, diff --git a/pkg/registry/apis/provisioning/webhooks/webhook.go b/pkg/registry/apis/provisioning/webhooks/webhook.go index 321f08fdafb..4e5bfffb28a 100644 --- a/pkg/registry/apis/provisioning/webhooks/webhook.go +++ b/pkg/registry/apis/provisioning/webhooks/webhook.go @@ -36,6 +36,7 @@ type webhookConnector struct { core *provisioningapis.APIBuilder renderer pullrequest.ScreenshotRenderer registry prometheus.Registerer + metrics webhookMetrics } func NewWebhookConnector( @@ -45,11 +46,13 @@ func NewWebhookConnector( renderer pullrequest.ScreenshotRenderer, registry prometheus.Registerer, ) *webhookConnector { + metrics := registerWebhookMetrics(registry) return &webhookConnector{ webhooksEnabled: webhooksEnabled, core: core, renderer: renderer, registry: registry, + metrics: metrics, } } @@ -151,10 +154,17 @@ func (s *webhookConnector) Connect(ctx context.Context, name string, opts runtim logger.Error("failed to update last event", "error", err) } + actionTaken := "none" + defer func() { + s.metrics.recordEventProcessed(actionTaken) + }() + if rsp.Job != nil { rsp.Job.Repository = name + actionTaken = string(rsp.Job.Action) job, err := s.core.GetJobQueue().Insert(ctx, namespace, *rsp.Job) if err != nil { + logger.Error("failed to insert job", "error", err) responder.Error(err) return } diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index fa09706c01c..4a33e8e41b9 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -829,7 +829,7 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, dashboardPermissionsService, accessControl, featureToggles) webhookExtraBuilder := webhooks.ProvideWebhooksWithImages(cfg, renderingService, resourceClient, eventualRestConfigProvider, registerer) v3 := extras.ProvideProvisioningExtraAPIs(webhookExtraBuilder) - pullRequestWorker := pullrequest.ProvidePullRequestWorker(cfg, renderingService, resourceClient, eventualRestConfigProvider) + pullRequestWorker := pullrequest.ProvidePullRequestWorker(cfg, renderingService, resourceClient, eventualRestConfigProvider, registerer) v4 := extras.ProvideExtraWorkers(pullRequestWorker) v5 := _wireValue decryptAuthorizer := decrypt.ProvideDecryptAuthorizer(tracer, v5) @@ -1434,7 +1434,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, dashboardPermissionsService, accessControl, featureToggles) webhookExtraBuilder := webhooks.ProvideWebhooksWithImages(cfg, renderingService, resourceClient, eventualRestConfigProvider, registerer) v3 := extras.ProvideProvisioningExtraAPIs(webhookExtraBuilder) - pullRequestWorker := pullrequest.ProvidePullRequestWorker(cfg, renderingService, resourceClient, eventualRestConfigProvider) + pullRequestWorker := pullrequest.ProvidePullRequestWorker(cfg, renderingService, resourceClient, eventualRestConfigProvider, registerer) v4 := extras.ProvideExtraWorkers(pullRequestWorker) v5 := _wireValue decryptAuthorizer := decrypt.ProvideDecryptAuthorizer(tracer, v5) From e53b5902fae548fedc637900f4af86ca34301f91 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Mon, 22 Sep 2025 22:23:29 -0600 Subject: [PATCH 25/54] Provisioning: Wire up tracing and add trace for job sync (#111455) --- pkg/operators/provisioning/jobs_operator.go | 18 ++++- pkg/operators/provisioning/repo_operator.go | 12 ++++ .../provisioning/controller/repository.go | 4 ++ .../apis/provisioning/jobs/sync/full.go | 61 +++++++++++----- .../jobs/sync/full_sync_fn_mock.go | 23 +++--- .../apis/provisioning/jobs/sync/full_test.go | 15 ++-- .../provisioning/jobs/sync/incremental.go | 45 ++++++++---- .../jobs/sync/incremental_sync_fn_mock.go | 23 +++--- .../jobs/sync/incremental_test.go | 5 +- .../apis/provisioning/jobs/sync/sync.go | 13 ++-- .../apis/provisioning/jobs/sync/sync_test.go | 8 ++- .../apis/provisioning/jobs/sync/worker.go | 72 +++++++++++++++---- .../provisioning/jobs/sync/worker_test.go | 6 +- pkg/registry/apis/provisioning/register.go | 4 +- 14 files changed, 223 insertions(+), 86 deletions(-) diff --git a/pkg/operators/provisioning/jobs_operator.go b/pkg/operators/provisioning/jobs_operator.go index fb9fd9c494e..f901988e0e0 100644 --- a/pkg/operators/provisioning/jobs_operator.go +++ b/pkg/operators/provisioning/jobs_operator.go @@ -13,6 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/tools/cache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/migrate" @@ -34,6 +35,16 @@ func RunJobController(deps server.OperatorDependencies) error { })).With("logger", "provisioning-job-controller") logger.Info("Starting provisioning job controller") + tracingConfig, err := tracing.ProvideTracingConfig(deps.Config) + if err != nil { + return fmt.Errorf("failed to provide tracing config: %w", err) + } + + tracer, err := tracing.ProvideService(tracingConfig) + if err != nil { + return fmt.Errorf("failed to provide tracing service: %w", err) + } + controllerCfg, err := setupJobsControllerFromConfig(deps.Config, deps.Registerer) if err != nil { return fmt.Errorf("failed to setup operator: %w", err) @@ -100,7 +111,7 @@ func RunJobController(deps server.OperatorDependencies) error { return fmt.Errorf("create API client job store: %w", err) } - workers, err := setupWorkers(controllerCfg, deps.Registerer) + workers, err := setupWorkers(controllerCfg, deps.Registerer, tracer) if err != nil { return fmt.Errorf("setup workers: %w", err) } @@ -175,7 +186,7 @@ func setupJobsControllerFromConfig(cfg *setting.Cfg, registry prometheus.Registe }, nil } -func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Registerer) ([]jobs.Worker, error) { +func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Registerer, tracer tracing.Tracer) ([]jobs.Worker, error) { clients := controllerCfg.clients parsers := resources.NewParserFactory(clients) resourceLister := resources.NewResourceLister(controllerCfg.unified) @@ -187,7 +198,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis metrics := jobs.RegisterJobMetrics(registry) // Sync - syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync) + syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync, tracer) syncWorker := sync.NewSyncWorker( clients, repositoryResources, @@ -195,6 +206,7 @@ func setupWorkers(controllerCfg *jobsControllerConfig, registry prometheus.Regis statusPatcher.Patch, syncer, metrics, + tracer, ) workers = append(workers, syncWorker) diff --git a/pkg/operators/provisioning/repo_operator.go b/pkg/operators/provisioning/repo_operator.go index a15c2033d57..0ec243bb41e 100644 --- a/pkg/operators/provisioning/repo_operator.go +++ b/pkg/operators/provisioning/repo_operator.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/tools/cache" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/controller" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" @@ -34,6 +35,16 @@ func RunRepoController(deps server.OperatorDependencies) error { return fmt.Errorf("failed to setup operator: %w", err) } + tracingConfig, err := tracing.ProvideTracingConfig(deps.Config) + if err != nil { + return fmt.Errorf("failed to provide tracing config: %w", err) + } + + tracer, err := tracing.ProvideService(tracingConfig) + if err != nil { + return fmt.Errorf("failed to provide tracing service: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -70,6 +81,7 @@ func RunRepoController(deps server.OperatorDependencies) error { healthChecker, statusPatcher, deps.Registerer, + tracer, ) if err != nil { return fmt.Errorf("failed to create repository controller: %w", err) diff --git a/pkg/registry/apis/provisioning/controller/repository.go b/pkg/registry/apis/provisioning/controller/repository.go index cbc55781819..ccda76e1722 100644 --- a/pkg/registry/apis/provisioning/controller/repository.go +++ b/pkg/registry/apis/provisioning/controller/repository.go @@ -22,6 +22,7 @@ import ( listers "github.com/grafana/grafana/apps/provisioning/pkg/generated/listers/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" @@ -62,6 +63,7 @@ type RepositoryController struct { queue workqueue.TypedRateLimitingInterface[*queueItem] registry prometheus.Registerer + tracer tracing.Tracer } // NewRepositoryController creates new RepositoryController. @@ -76,6 +78,7 @@ func NewRepositoryController( healthChecker *HealthChecker, statusPatcher StatusPatcher, registry prometheus.Registerer, + tracer tracing.Tracer, ) (*RepositoryController, error) { finalizerMetrics := registerFinalizerMetrics(registry) @@ -101,6 +104,7 @@ func NewRepositoryController( logger: logging.DefaultLogger.With("logger", loggerName), dualwrite: dualwrite, registry: registry, + tracer: tracer, } _, err := repoInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/pkg/registry/apis/provisioning/jobs/sync/full.go b/pkg/registry/apis/provisioning/jobs/sync/full.go index 8114d8a67ce..0e0b107171a 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/full.go +++ b/pkg/registry/apis/provisioning/jobs/sync/full.go @@ -6,8 +6,11 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/apps/provisioning/pkg/safepath" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -20,47 +23,63 @@ func FullSync( currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, + tracer tracing.Tracer, ) error { cfg := repo.Config() + ctx, span := tracer.Start(ctx, "provisioning.sync.full") + defer span.End() + + ensureFolderCtx, ensureFolderSpan := tracer.Start(ctx, "provisioning.sync.full.ensure_folder_exists") // Ensure the configured folder exists and is managed by the repository rootFolder := resources.RootFolder(cfg) if rootFolder != "" { - if err := repositoryResources.EnsureFolderExists(ctx, resources.Folder{ + if err := repositoryResources.EnsureFolderExists(ensureFolderCtx, resources.Folder{ ID: rootFolder, // will not change if exists Title: cfg.Spec.Title, Path: "", // at the root of the repository }, ""); err != nil { - return fmt.Errorf("create root folder: %w", err) + ensureFolderSpan.End() + return tracing.Error(span, fmt.Errorf("create root folder: %w", err)) } } + ensureFolderSpan.End() - changes, err := compare(ctx, repo, repositoryResources, currentRef) + compareCtx, compareSpan := tracer.Start(ctx, "provisioning.sync.full.compare") + changes, err := compare(compareCtx, repo, repositoryResources, currentRef) if err != nil { - return fmt.Errorf("compare changes: %w", err) + compareSpan.End() + return tracing.Error(span, fmt.Errorf("compare changes: %w", err)) } + compareSpan.End() if len(changes) == 0 { progress.SetFinalMessage(ctx, "no changes to sync") return nil } - return applyChanges(ctx, changes, clients, repositoryResources, progress) + return applyChanges(ctx, changes, clients, repositoryResources, progress, tracer) } -func applyChanges(ctx context.Context, changes []ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { +func applyChanges(ctx context.Context, changes []ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error { progress.SetTotal(ctx, len(changes)) + _, applyChangesSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes", + trace.WithAttributes(attribute.Int("changes_count", len(changes))), + ) + defer applyChangesSpan.End() + for _, change := range changes { if ctx.Err() != nil { return ctx.Err() } if err := progress.TooManyErrors(); err != nil { - return err + return tracing.Error(applyChangesSpan, err) } if change.Action == repository.FileActionDeleted { + deleteCtx, deleteSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes.delete") result := jobs.JobResourceResult{ Path: change.Path, Action: change.Action, @@ -68,7 +87,9 @@ func applyChanges(ctx context.Context, changes []ResourceFileChange, clients res if change.Existing == nil || change.Existing.Name == "" { result.Error = fmt.Errorf("processing deletion for file %s: missing existing reference", change.Path) - progress.Record(ctx, result) + progress.Record(deleteCtx, result) + deleteSpan.RecordError(result.Error) + deleteSpan.End() continue } @@ -82,22 +103,24 @@ func applyChanges(ctx context.Context, changes []ResourceFileChange, clients res } // TODO: should we use the clients or the resource manager instead? - client, _, err := clients.ForResource(ctx, versionlessGVR) + client, _, err := clients.ForResource(deleteCtx, versionlessGVR) if err != nil { result.Error = fmt.Errorf("get client for deleted object: %w", err) - progress.Record(ctx, result) + progress.Record(deleteCtx, result) continue } - if err := client.Delete(ctx, change.Existing.Name, metav1.DeleteOptions{}); err != nil { + if err := client.Delete(deleteCtx, change.Existing.Name, metav1.DeleteOptions{}); err != nil { result.Error = fmt.Errorf("deleting resource %s/%s %s: %w", change.Existing.Group, change.Existing.Resource, change.Existing.Name, err) } - progress.Record(ctx, result) + progress.Record(deleteCtx, result) + deleteSpan.End() continue } // If folder ensure it exists if safepath.IsDir(change.Path) { + ensureFolderCtx, ensureFolderSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes.ensure_folder_exists") result := jobs.JobResourceResult{ Path: change.Path, Action: change.Action, @@ -105,20 +128,24 @@ func applyChanges(ctx context.Context, changes []ResourceFileChange, clients res Group: resources.FolderResource.Group, } - folder, err := repositoryResources.EnsureFolderPathExist(ctx, change.Path) + folder, err := repositoryResources.EnsureFolderPathExist(ensureFolderCtx, change.Path) if err != nil { result.Error = fmt.Errorf("ensuring folder exists at path %s: %w", change.Path, err) + ensureFolderSpan.RecordError(err) + ensureFolderSpan.End() progress.Record(ctx, result) continue } result.Name = folder - progress.Record(ctx, result) + progress.Record(ensureFolderCtx, result) + ensureFolderSpan.End() continue } - name, gvk, err := repositoryResources.WriteResourceFromFile(ctx, change.Path, "") + writeCtx, writeSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes.write_resource_from_file") + name, gvk, err := repositoryResources.WriteResourceFromFile(writeCtx, change.Path, "") result := jobs.JobResourceResult{ Path: change.Path, Action: change.Action, @@ -128,9 +155,11 @@ func applyChanges(ctx context.Context, changes []ResourceFileChange, clients res } if err != nil { + writeSpan.RecordError(err) result.Error = fmt.Errorf("writing resource from file %s: %w", change.Path, err) } - progress.Record(ctx, result) + progress.Record(writeCtx, result) + writeSpan.End() } return nil diff --git a/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go b/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go index 03b080cff25..3f10550e3f8 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go +++ b/pkg/registry/apis/provisioning/jobs/sync/full_sync_fn_mock.go @@ -11,6 +11,8 @@ import ( repository "github.com/grafana/grafana/apps/provisioning/pkg/repository" resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + + tracing "github.com/grafana/grafana/pkg/infra/tracing" ) // MockFullSyncFn is an autogenerated mock type for the FullSyncFn type @@ -26,17 +28,17 @@ func (_m *MockFullSyncFn) EXPECT() *MockFullSyncFn_Expecter { return &MockFullSyncFn_Expecter{mock: &_m.Mock} } -// Execute provides a mock function with given fields: ctx, repo, compare, clients, currentRef, repositoryResources, progress -func (_m *MockFullSyncFn) Execute(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { - ret := _m.Called(ctx, repo, compare, clients, currentRef, repositoryResources, progress) +// Execute provides a mock function with given fields: ctx, repo, compare, clients, currentRef, repositoryResources, progress, tracer +func (_m *MockFullSyncFn) Execute(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error { + ret := _m.Called(ctx, repo, compare, clients, currentRef, repositoryResources, progress, tracer) if len(ret) == 0 { panic("no return value specified for Execute") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder) error); ok { - r0 = rf(ctx, repo, compare, clients, currentRef, repositoryResources, progress) + if rf, ok := ret.Get(0).(func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder, tracing.Tracer) error); ok { + r0 = rf(ctx, repo, compare, clients, currentRef, repositoryResources, progress, tracer) } else { r0 = ret.Error(0) } @@ -57,13 +59,14 @@ type MockFullSyncFn_Execute_Call struct { // - currentRef string // - repositoryResources resources.RepositoryResources // - progress jobs.JobProgressRecorder -func (_e *MockFullSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, compare interface{}, clients interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}) *MockFullSyncFn_Execute_Call { - return &MockFullSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, compare, clients, currentRef, repositoryResources, progress)} +// - tracer tracing.Tracer +func (_e *MockFullSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, compare interface{}, clients interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}, tracer interface{}) *MockFullSyncFn_Execute_Call { + return &MockFullSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, compare, clients, currentRef, repositoryResources, progress, tracer)} } -func (_c *MockFullSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder)) *MockFullSyncFn_Execute_Call { +func (_c *MockFullSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer)) *MockFullSyncFn_Execute_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(repository.Reader), args[2].(CompareFn), args[3].(resources.ResourceClients), args[4].(string), args[5].(resources.RepositoryResources), args[6].(jobs.JobProgressRecorder)) + run(args[0].(context.Context), args[1].(repository.Reader), args[2].(CompareFn), args[3].(resources.ResourceClients), args[4].(string), args[5].(resources.RepositoryResources), args[6].(jobs.JobProgressRecorder), args[7].(tracing.Tracer)) }) return _c } @@ -73,7 +76,7 @@ func (_c *MockFullSyncFn_Execute_Call) Return(_a0 error) *MockFullSyncFn_Execute return _c } -func (_c *MockFullSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder) error) *MockFullSyncFn_Execute_Call { +func (_c *MockFullSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Reader, CompareFn, resources.ResourceClients, string, resources.RepositoryResources, jobs.JobProgressRecorder, tracing.Tracer) error) *MockFullSyncFn_Execute_Call { _c.Call.Return(run) return _c } diff --git a/pkg/registry/apis/provisioning/jobs/sync/full_test.go b/pkg/registry/apis/provisioning/jobs/sync/full_test.go index 0f051e20b72..d5b3b610b80 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/full_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/full_test.go @@ -8,6 +8,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/stretchr/testify/mock" @@ -41,7 +42,7 @@ func TestFullSync_ContextCancelled(t *testing.T) { compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{{}}, nil) progress.On("SetTotal", mock.Anything, 1).Return() - err := FullSync(ctx, repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(ctx, repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) require.EqualError(t, err, "context canceled") } @@ -60,7 +61,7 @@ func TestFullSync_Error(t *testing.T) { compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("some error")) - err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) require.EqualError(t, err, "compare changes: some error") } @@ -80,7 +81,7 @@ func TestFullSync_NoChanges(t *testing.T) { compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]ResourceFileChange{}, nil) progress.On("SetFinalMessage", mock.Anything, "no changes to sync").Return() - err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) require.NoError(t, err) } @@ -111,7 +112,7 @@ func TestFullSync_SuccessfulFolderCreation(t *testing.T) { Path: "", }, "").Return(nil) - err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) require.NoError(t, err) } @@ -140,7 +141,7 @@ func TestFullSync_FolderCreationFailed(t *testing.T) { Path: "", }, "").Return(fmt.Errorf("folder creation failed")) - err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) require.Error(t, err) require.Contains(t, err.Error(), "create root folder: folder creation failed") } @@ -169,7 +170,7 @@ func TestFullSync_FolderCreationFailedWithInstanceTarget(t *testing.T) { compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(nil, fmt.Errorf("compare error")) - err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) require.Error(t, err) require.Contains(t, err.Error(), "compare changes: compare error") } @@ -700,7 +701,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo }) progress.On("SetTotal", mock.Anything, len(tt.changes)).Return() - err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress) + err := FullSync(context.Background(), repo, compareFn.Execute, clients, "current-ref", repoResources, progress, tracing.NewNoopTracerService()) if tt.expectedError != "" { require.EqualError(t, err, tt.expectedError, tt.description) } else { diff --git a/pkg/registry/apis/provisioning/jobs/sync/incremental.go b/pkg/registry/apis/provisioning/jobs/sync/incremental.go index fe66ad058ee..f7448bffa16 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/incremental.go +++ b/pkg/registry/apis/provisioning/jobs/sync/incremental.go @@ -6,21 +6,29 @@ import ( "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/apps/provisioning/pkg/safepath" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) // Convert git changes into resource file changes -func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { +func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error { if previousRef == currentRef { progress.SetFinalMessage(ctx, "same commit as last time") return nil } - diff, err := repo.CompareFiles(ctx, previousRef, currentRef) + ctx, span := tracer.Start(ctx, "provisioning.sync.incremental") + defer span.End() + + compareCtx, compareSpan := tracer.Start(ctx, "provisioning.sync.incremental.compare_files") + diff, err := repo.CompareFiles(compareCtx, previousRef, currentRef) if err != nil { - return fmt.Errorf("compare files error: %w", err) + compareSpan.RecordError(err) + compareSpan.End() + return tracing.Error(span, fmt.Errorf("compare files error: %w", err)) } + compareSpan.End() if len(diff) < 1 { progress.SetFinalMessage(ctx, "no changes detected between commits") @@ -35,10 +43,11 @@ func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef return ctx.Err() } if err := progress.TooManyErrors(); err != nil { - return err + return tracing.Error(span, err) } if err := resources.IsPathSupported(change.Path); err != nil { + ensureFolderCtx, ensureFolderSpan := tracer.Start(ctx, "provisioning.sync.incremental.ensure_folder_path_exist") // Maintain the safe segment for empty folders safeSegment := safepath.SafeSegment(change.Path) if !safepath.IsDir(safeSegment) { @@ -46,26 +55,29 @@ func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef } if safeSegment != "" && resources.IsPathSupported(safeSegment) == nil { - folder, err := repositoryResources.EnsureFolderPathExist(ctx, safeSegment) + folder, err := repositoryResources.EnsureFolderPathExist(ensureFolderCtx, safeSegment) if err != nil { - return fmt.Errorf("unable to create empty file folder: %w", err) + ensureFolderSpan.RecordError(err) + ensureFolderSpan.End() + return tracing.Error(span, fmt.Errorf("unable to create empty file folder: %w", err)) } - progress.Record(ctx, jobs.JobResourceResult{ + progress.Record(ensureFolderCtx, jobs.JobResourceResult{ Path: safeSegment, Action: repository.FileActionCreated, Resource: resources.FolderResource.Resource, Group: resources.FolderResource.Group, Name: folder, }) - + ensureFolderSpan.End() continue } - progress.Record(ctx, jobs.JobResourceResult{ + progress.Record(ensureFolderCtx, jobs.JobResourceResult{ Path: change.Path, Action: repository.FileActionIgnored, }) + ensureFolderSpan.End() continue } @@ -76,29 +88,38 @@ func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef switch change.Action { case repository.FileActionCreated, repository.FileActionUpdated: - name, gvk, err := repositoryResources.WriteResourceFromFile(ctx, change.Path, change.Ref) + writeCtx, writeSpan := tracer.Start(ctx, "provisioning.sync.incremental.write_resource_from_file") + name, gvk, err := repositoryResources.WriteResourceFromFile(writeCtx, change.Path, change.Ref) if err != nil { + writeSpan.RecordError(err) result.Error = fmt.Errorf("writing resource from file %s: %w", change.Path, err) } result.Name = name result.Resource = gvk.Kind result.Group = gvk.Group + writeSpan.End() case repository.FileActionDeleted: - name, gvk, err := repositoryResources.RemoveResourceFromFile(ctx, change.Path, change.PreviousRef) + removeCtx, removeSpan := tracer.Start(ctx, "provisioning.sync.incremental.remove_resource_from_file") + name, gvk, err := repositoryResources.RemoveResourceFromFile(removeCtx, change.Path, change.PreviousRef) if err != nil { + removeSpan.RecordError(err) result.Error = fmt.Errorf("removing resource from file %s: %w", change.Path, err) } result.Name = name result.Resource = gvk.Kind result.Group = gvk.Group + removeSpan.End() case repository.FileActionRenamed: - name, gvk, err := repositoryResources.RenameResourceFile(ctx, change.PreviousPath, change.PreviousRef, change.Path, change.Ref) + renameCtx, renameSpan := tracer.Start(ctx, "provisioning.sync.incremental.rename_resource_file") + name, gvk, err := repositoryResources.RenameResourceFile(renameCtx, change.PreviousPath, change.PreviousRef, change.Path, change.Ref) if err != nil { + renameSpan.RecordError(err) result.Error = fmt.Errorf("renaming resource file from %s to %s: %w", change.PreviousPath, change.Path, err) } result.Name = name result.Resource = gvk.Kind result.Group = gvk.Group + renameSpan.End() case repository.FileActionIgnored: // do nothing } diff --git a/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go b/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go index c1ae1325475..ff032146788 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go +++ b/pkg/registry/apis/provisioning/jobs/sync/incremental_sync_fn_mock.go @@ -11,6 +11,8 @@ import ( repository "github.com/grafana/grafana/apps/provisioning/pkg/repository" resources "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" + + tracing "github.com/grafana/grafana/pkg/infra/tracing" ) // MockIncrementalSyncFn is an autogenerated mock type for the IncrementalSyncFn type @@ -26,17 +28,17 @@ func (_m *MockIncrementalSyncFn) EXPECT() *MockIncrementalSyncFn_Expecter { return &MockIncrementalSyncFn_Expecter{mock: &_m.Mock} } -// Execute provides a mock function with given fields: ctx, repo, previousRef, currentRef, repositoryResources, progress -func (_m *MockIncrementalSyncFn) Execute(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error { - ret := _m.Called(ctx, repo, previousRef, currentRef, repositoryResources, progress) +// Execute provides a mock function with given fields: ctx, repo, previousRef, currentRef, repositoryResources, progress, tracer +func (_m *MockIncrementalSyncFn) Execute(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error { + ret := _m.Called(ctx, repo, previousRef, currentRef, repositoryResources, progress, tracer) if len(ret) == 0 { panic("no return value specified for Execute") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder) error); ok { - r0 = rf(ctx, repo, previousRef, currentRef, repositoryResources, progress) + if rf, ok := ret.Get(0).(func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder, tracing.Tracer) error); ok { + r0 = rf(ctx, repo, previousRef, currentRef, repositoryResources, progress, tracer) } else { r0 = ret.Error(0) } @@ -56,13 +58,14 @@ type MockIncrementalSyncFn_Execute_Call struct { // - currentRef string // - repositoryResources resources.RepositoryResources // - progress jobs.JobProgressRecorder -func (_e *MockIncrementalSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, previousRef interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}) *MockIncrementalSyncFn_Execute_Call { - return &MockIncrementalSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, previousRef, currentRef, repositoryResources, progress)} +// - tracer tracing.Tracer +func (_e *MockIncrementalSyncFn_Expecter) Execute(ctx interface{}, repo interface{}, previousRef interface{}, currentRef interface{}, repositoryResources interface{}, progress interface{}, tracer interface{}) *MockIncrementalSyncFn_Execute_Call { + return &MockIncrementalSyncFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, previousRef, currentRef, repositoryResources, progress, tracer)} } -func (_c *MockIncrementalSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder)) *MockIncrementalSyncFn_Execute_Call { +func (_c *MockIncrementalSyncFn_Execute_Call) Run(run func(ctx context.Context, repo repository.Versioned, previousRef string, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer)) *MockIncrementalSyncFn_Execute_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(repository.Versioned), args[2].(string), args[3].(string), args[4].(resources.RepositoryResources), args[5].(jobs.JobProgressRecorder)) + run(args[0].(context.Context), args[1].(repository.Versioned), args[2].(string), args[3].(string), args[4].(resources.RepositoryResources), args[5].(jobs.JobProgressRecorder), args[6].(tracing.Tracer)) }) return _c } @@ -72,7 +75,7 @@ func (_c *MockIncrementalSyncFn_Execute_Call) Return(_a0 error) *MockIncremental return _c } -func (_c *MockIncrementalSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder) error) *MockIncrementalSyncFn_Execute_Call { +func (_c *MockIncrementalSyncFn_Execute_Call) RunAndReturn(run func(context.Context, repository.Versioned, string, string, resources.RepositoryResources, jobs.JobProgressRecorder, tracing.Tracer) error) *MockIncrementalSyncFn_Execute_Call { _c.Call.Return(run) return _c } diff --git a/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go b/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go index 290b40c77ce..6562be15f3c 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/incremental_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/stretchr/testify/mock" @@ -29,7 +30,7 @@ func TestIncrementalSync_ContextCancelled(t *testing.T) { progress.On("SetTotal", mock.Anything, 1).Return() progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return() - err := IncrementalSync(ctx, repo, "old-ref", "new-ref", repoResources, progress) + err := IncrementalSync(ctx, repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService()) require.EqualError(t, err, "context canceled") } @@ -386,7 +387,7 @@ func TestIncrementalSync(t *testing.T) { tt.setupMocks(repo, repoResources, progress) - err := IncrementalSync(context.Background(), repo, tt.previousRef, tt.currentRef, repoResources, progress) + err := IncrementalSync(context.Background(), repo, tt.previousRef, tt.currentRef, repoResources, progress, tracing.NewNoopTracerService()) if tt.expectedError != "" { require.EqualError(t, err, tt.expectedError) diff --git a/pkg/registry/apis/provisioning/jobs/sync/sync.go b/pkg/registry/apis/provisioning/jobs/sync/sync.go index 541fd47a5f9..fa58d426cba 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/sync.go +++ b/pkg/registry/apis/provisioning/jobs/sync/sync.go @@ -6,18 +6,19 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) //go:generate mockery --name FullSyncFn --structname MockFullSyncFn --inpackage --filename full_sync_fn_mock.go --with-expecter -type FullSyncFn func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error +type FullSyncFn func(ctx context.Context, repo repository.Reader, compare CompareFn, clients resources.ResourceClients, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error //go:generate mockery --name CompareFn --structname MockCompareFn --inpackage --filename compare_fn_mock.go --with-expecter type CompareFn func(ctx context.Context, repo repository.Reader, repositoryResources resources.RepositoryResources, ref string) ([]ResourceFileChange, error) //go:generate mockery --name IncrementalSyncFn --structname MockIncrementalSyncFn --inpackage --filename incremental_sync_fn_mock.go --with-expecter -type IncrementalSyncFn func(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder) error +type IncrementalSyncFn func(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error //go:generate mockery --name Syncer --structname MockSyncer --inpackage --filename syncer_mock.go --with-expecter type Syncer interface { @@ -28,13 +29,15 @@ type syncer struct { compare CompareFn fullSync FullSyncFn incrementalSync IncrementalSyncFn + tracer tracing.Tracer } -func NewSyncer(compare CompareFn, fullSync FullSyncFn, incrementalSync IncrementalSyncFn) Syncer { +func NewSyncer(compare CompareFn, fullSync FullSyncFn, incrementalSync IncrementalSyncFn, tracer tracing.Tracer) Syncer { return &syncer{ compare: compare, fullSync: fullSync, incrementalSync: incrementalSync, + tracer: tracer, } } @@ -52,11 +55,11 @@ func (r *syncer) Sync(ctx context.Context, repo repository.ReaderWriter, options if cfg.Status.Sync.LastRef != "" && options.Incremental { progress.SetMessage(ctx, "incremental sync") - return currentRef, r.incrementalSync(ctx, versionedRepo, cfg.Status.Sync.LastRef, currentRef, repositoryResources, progress) + return currentRef, r.incrementalSync(ctx, versionedRepo, cfg.Status.Sync.LastRef, currentRef, repositoryResources, progress, r.tracer) } } progress.SetMessage(ctx, "full sync") - return currentRef, r.fullSync(ctx, repo, r.compare, clients, currentRef, repositoryResources, progress) + return currentRef, r.fullSync(ctx, repo, r.compare, clients, currentRef, repositoryResources, progress, r.tracer) } diff --git a/pkg/registry/apis/provisioning/jobs/sync/sync_test.go b/pkg/registry/apis/provisioning/jobs/sync/sync_test.go index 1751159659a..5f04ed350e1 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/sync_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/sync_test.go @@ -7,6 +7,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" mock "github.com/stretchr/testify/mock" @@ -65,7 +66,7 @@ func TestSyncer_Sync(t *testing.T) { repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil) progress.On("SetMessage", mock.Anything, "full sync").Return() - fullSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, mock.Anything, mock.Anything, "new-ref", mock.Anything, mock.Anything).Return(nil) + fullSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, mock.Anything, mock.Anything, "new-ref", mock.Anything, mock.Anything, mock.Anything).Return(nil) }, expectedMessages: []string{"full sync"}, }, @@ -87,7 +88,7 @@ func TestSyncer_Sync(t *testing.T) { }) repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil) progress.On("SetMessage", mock.Anything, "incremental sync").Return() - incrementalSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything).Return(nil) + incrementalSyncFn.EXPECT().Execute(mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything, mock.Anything).Return(nil) }, expectedRef: "new-ref", expectedMessages: []string{"incremental sync"}, @@ -130,7 +131,7 @@ func TestSyncer_Sync(t *testing.T) { }) repo.MockVersioned.On("LatestRef", mock.Anything).Return("new-ref", nil) progress.On("SetMessage", mock.Anything, "incremental sync").Return() - incrementalSyncFn.On("Execute", mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything).Return(fmt.Errorf("incremental sync failed")) + incrementalSyncFn.On("Execute", mock.Anything, mock.Anything, "old-ref", "new-ref", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("incremental sync failed")) }, expectedRef: "new-ref", expectedMessages: []string{"incremental sync"}, @@ -158,6 +159,7 @@ func TestSyncer_Sync(t *testing.T) { compareFn.Execute, fullSyncFn.Execute, incrementalSyncFn.Execute, + tracing.NewNoopTracerService(), ) ref, err := syncer.Sync(context.Background(), repo, tt.options, repoResources, clients, progress) diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker.go b/pkg/registry/apis/provisioning/jobs/sync/worker.go index 5443c814bcf..0b0139941c0 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker.go @@ -8,10 +8,13 @@ import ( "github.com/grafana/grafana-app-sdk/logging" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) //go:generate mockery --name RepositoryPatchFn --structname MockRepositoryPatchFn --inpackage --filename repository_patch_fn_mock.go --with-expecter @@ -36,6 +39,8 @@ type SyncWorker struct { syncer Syncer metrics jobs.JobMetrics + + tracer tracing.Tracer } func NewSyncWorker( @@ -45,6 +50,7 @@ func NewSyncWorker( patchStatus RepositoryPatchFn, syncer Syncer, metrics jobs.JobMetrics, + tracer tracing.Tracer, ) *SyncWorker { return &SyncWorker{ clients: clients, @@ -53,6 +59,7 @@ func NewSyncWorker( storageStatus: storageStatus, syncer: syncer, metrics: metrics, + tracer: tracer, } } @@ -63,23 +70,39 @@ func (r *SyncWorker) IsSupported(ctx context.Context, job provisioning.Job) bool func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, job provisioning.Job, progress jobs.JobProgressRecorder) error { cfg := repo.Config() logger := logging.FromContext(ctx).With("job", job.GetName(), "namespace", job.GetNamespace()) + ctx, span := r.tracer.Start(ctx, "provisioning.sync.process", + trace.WithAttributes( + attribute.String("job.name", job.GetName()), + attribute.String("job.namespace", job.GetNamespace()), + attribute.String("job.action", string(job.Spec.Action)), + attribute.String("repository.name", cfg.Name), + attribute.String("repository.namespace", cfg.Namespace), + ), + ) + defer span.End() start := time.Now() outcome := utils.ErrorOutcome totalChangesMade := 0 defer func() { r.metrics.RecordJob(string(provisioning.JobActionPull), outcome, totalChangesMade, time.Since(start).Seconds()) + span.SetAttributes( + attribute.String("outcome", outcome), + attribute.Int("changes_made", totalChangesMade), + ) }() // Check if we are onboarding from legacy storage // HACK -- this should be handled outside of this worker if r.storageStatus != nil && dualwrite.IsReadingLegacyDashboardsAndFolders(ctx, r.storageStatus) { - return fmt.Errorf("sync not supported until storage has migrated") + err := fmt.Errorf("sync not supported until storage has migrated") + return tracing.Error(span, err) } rw, ok := repo.(repository.ReaderWriter) if !ok { - return fmt.Errorf("sync job submitted for repository that does not support read-write") + err := fmt.Errorf("sync job submitted for repository that does not support read-write") + return tracing.Error(span, err) } syncStatus := job.Status.ToSyncStatus(job.Name) @@ -97,38 +120,49 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo } progress.SetMessage(ctx, "update sync status at start") - if err := r.patchStatus(ctx, cfg, patchOperations...); err != nil { + + statusCtx, statusSpan := r.tracer.Start(ctx, "provisioning.sync.update_start_status") + if err := r.patchStatus(statusCtx, cfg, patchOperations...); err != nil { + statusSpan.End() logger.Error("failed to update the repository status at the start of the sync job", "error", err) - return fmt.Errorf("update repo with job status at start: %w", err) + err = fmt.Errorf("update repo with job status at start: %w", err) + return tracing.Error(span, err) } + statusSpan.End() - repositoryResources, err := r.repositoryResources.Client(ctx, rw) + setupCtx, setupSpan := r.tracer.Start(ctx, "provisioning.sync.setup_clients") + repositoryResources, err := r.repositoryResources.Client(setupCtx, rw) if err != nil { + setupSpan.End() logger.Error("failed to create repository resources client", "error", err) - return fmt.Errorf("create repository resources client: %w", err) + err = fmt.Errorf("create repository resources client: %w", err) + return tracing.Error(span, err) } - - clients, err := r.clients.Clients(ctx, cfg.Namespace) + clients, err := r.clients.Clients(setupCtx, cfg.Namespace) if err != nil { + setupSpan.End() logger.Error("failed to get clients for the repository", "error", err) - return fmt.Errorf("get clients for %s: %w", cfg.Name, err) + err = fmt.Errorf("get clients for %s: %w", cfg.Name, err) + return tracing.Error(span, err) } + setupSpan.End() + syncCtx, syncSpan := r.tracer.Start(ctx, "provisioning.sync.execute") progress.SetMessage(ctx, "execute sync job") progress.StrictMaxErrors(20) // make it stop after 20 errors - - currentRef, syncError := r.syncer.Sync(ctx, rw, *job.Spec.Pull, repositoryResources, clients, progress) + currentRef, syncError := r.syncer.Sync(syncCtx, rw, *job.Spec.Pull, repositoryResources, clients, progress) jobStatus := progress.Complete(ctx, syncError) syncStatus = jobStatus.ToSyncStatus(job.Name) - if syncError != nil { logger.Debug("failed to sync the repository", "error", syncError) + _ = tracing.Error(syncSpan, syncError) } else { outcome = utils.SuccessOutcome for _, summary := range jobStatus.Summary { totalChangesMade += int(summary.Create + summary.Update + summary.Delete) } } + syncSpan.End() // Create sync status and set hash if successful if syncStatus.State == provisioning.JobStateSuccess { @@ -147,13 +181,17 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo }, } + finalCtx, finalSpan := r.tracer.Start(ctx, "provisioning.sync.update_final_status") + // Only add stats patch if stats are not nil - stats, err := repositoryResources.Stats(ctx) + stats, err := repositoryResources.Stats(finalCtx) switch { case err != nil: logger.Error("unable to read stats", "error", err) + finalSpan.SetAttributes(attribute.String("stats.error", err.Error())) case stats == nil: logger.Error("stats are nil") + finalSpan.SetAttributes(attribute.Bool("stats.nil", true)) case len(stats.Managed) == 1: patchOperations = append(patchOperations, map[string]interface{}{ "op": "replace", @@ -162,13 +200,17 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo }) default: logger.Warn("unexpected number of managed stats", "count", len(stats.Managed)) + finalSpan.SetAttributes(attribute.Int("stats.unexpected_count", len(stats.Managed))) } // Only patch the specific fields we want to update, not the entire status - if err := r.patchStatus(ctx, cfg, patchOperations...); err != nil { + if err := r.patchStatus(finalCtx, cfg, patchOperations...); err != nil { + finalSpan.End() logger.Error("failed to update the repository status at the end of the sync job", "error", err) - return fmt.Errorf("update repo with job final status: %w", err) + err = fmt.Errorf("update repo with job final status: %w", err) + return tracing.Error(span, err) } + finalSpan.End() return syncError } diff --git a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go index 87ea058735d..090d4deac87 100644 --- a/pkg/registry/apis/provisioning/jobs/sync/worker_test.go +++ b/pkg/registry/apis/provisioning/jobs/sync/worker_test.go @@ -7,6 +7,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/repository" + "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" "github.com/grafana/grafana/pkg/storage/legacysql/dualwrite" @@ -45,7 +46,7 @@ func TestSyncWorker_IsSupported(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - worker := NewSyncWorker(nil, nil, nil, nil, nil, metrics) + worker := NewSyncWorker(nil, nil, nil, nil, nil, metrics, tracing.NewNoopTracerService()) result := worker.IsSupported(context.Background(), tt.job) require.Equal(t, tt.expected, result) }) @@ -64,7 +65,7 @@ func TestSyncWorker_ProcessNotReaderWriter(t *testing.T) { }) fakeDualwrite := dualwrite.NewMockService(t) fakeDualwrite.On("ReadFromUnified", mock.Anything, mock.Anything).Return(true, nil).Twice() - worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry())) + worker := NewSyncWorker(nil, nil, fakeDualwrite, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()), tracing.NewNoopTracerService()) err := worker.Process(context.Background(), repo, provisioning.Job{}, jobs.NewMockJobProgressRecorder(t)) require.EqualError(t, err, "sync job submitted for repository that does not support read-write") } @@ -533,6 +534,7 @@ func TestSyncWorker_Process(t *testing.T) { repositoryPatchFn.Execute, syncer, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()), + tracing.NewNoopTracerService(), ) // Create test job diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index 1904c417071..920c8680b21 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -748,7 +748,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH metrics, ) - syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync) + syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync, b.tracer) syncWorker := sync.NewSyncWorker( b.clients, b.repositoryResources, @@ -756,6 +756,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.statusPatcher.Patch, syncer, metrics, + b.tracer, ) signerFactory := signature.NewSignerFactory(b.clients) legacyResources := migrate.NewLegacyResourcesMigrator( @@ -847,6 +848,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH b.GetHealthChecker(), b.statusPatcher, b.registry, + b.tracer, ) if err != nil { return err From d471817cc72f27776f7115b90fbbfeb02f229efc Mon Sep 17 00:00:00 2001 From: Pepe Cano <825430+ppcano@users.noreply.github.com> Date: Tue, 23 Sep 2025 10:29:02 +0200 Subject: [PATCH 26/54] ui(alerting): Update "Enter custom fieldName" placeholder (#109665) * ui(alerting): Update custom template placeholder * Use i18n unescape format, instead of JS unescape fn * Run `make i18n-extract` * edit code comment --- .../receivers/form/fields/TemplateSelector.tsx | 14 +++++++++++++- public/locales/en-US/grafana.json | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.tsx b/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.tsx index 2af4f973bed..76dbd89b9e2 100644 --- a/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.tsx +++ b/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.tsx @@ -303,7 +303,19 @@ function OptionCustomfield({