Compare commits
34 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c8da64e4eb | |||
| 2fab497c18 | |||
| b0bb71f834 | |||
| a7aa55f908 | |||
| 0a61846b5e | |||
| 785cc739ee | |||
| 4913baaf04 | |||
| c8f1efe7c7 | |||
| b91ca14f48 | |||
| 2aedbdb76f | |||
| 0d7f46c08a | |||
| 1b52718c23 | |||
| e61e406440 | |||
| 5cb4c311dc | |||
| 586410d8b5 | |||
| a0e894c6d8 | |||
| e4796b1de3 | |||
| 86a3aae204 | |||
| e0ad4eb7ed | |||
| f0c95a0a10 | |||
| 0b46123300 | |||
| 81b868ae91 | |||
| 2a6a48ac39 | |||
| f581a5a69b | |||
| 5ff18c0802 | |||
| 0c8c886930 | |||
| ba196958cd | |||
| 622c75af6d | |||
| e983aac141 | |||
| c42421e616 | |||
| e4ba1c1a6d | |||
| 909ed02218 | |||
| ad3763f04d | |||
| ca6ab973b4 |
+22
-4
@@ -290,7 +290,7 @@
|
||||
],
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"placement": "right",
|
||||
"showLegend": true,
|
||||
"values": [
|
||||
"percent"
|
||||
@@ -304,7 +304,7 @@
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showLegend": false,
|
||||
"showLegend": true,
|
||||
"strokeWidth": 1,
|
||||
"text": {}
|
||||
},
|
||||
@@ -323,6 +323,15 @@
|
||||
}
|
||||
],
|
||||
"title": "Percent",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "renameByRegex",
|
||||
"options": {
|
||||
"regex": "^Backend-(.*)$",
|
||||
"renamePattern": "b-$1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
@@ -366,7 +375,7 @@
|
||||
],
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"placement": "right",
|
||||
"showLegend": true,
|
||||
"values": [
|
||||
"value"
|
||||
@@ -380,7 +389,7 @@
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showLegend": false,
|
||||
"showLegend": true,
|
||||
"strokeWidth": 1,
|
||||
"text": {}
|
||||
},
|
||||
@@ -399,6 +408,15 @@
|
||||
}
|
||||
],
|
||||
"title": "Value",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "renameByRegex",
|
||||
"options": {
|
||||
"regex": "(.*)",
|
||||
"renamePattern": "$1-how-much-wood-could-a-woodchuck-chuck-if-a-woodchuck-could-chuck-wood"
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -248,7 +248,7 @@
|
||||
"legend": {
|
||||
"values": ["percent"],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
"placement": "right"
|
||||
},
|
||||
"pieType": "pie",
|
||||
"reduceOptions": {
|
||||
@@ -256,7 +256,7 @@
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showLegend": false,
|
||||
"showLegend": true,
|
||||
"strokeWidth": 1,
|
||||
"text": {}
|
||||
},
|
||||
@@ -272,6 +272,15 @@
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Percent",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "renameByRegex",
|
||||
"options": {
|
||||
"regex": "^Backend-(.*)$",
|
||||
"renamePattern": "b-$1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
@@ -311,7 +320,7 @@
|
||||
"legend": {
|
||||
"values": ["value"],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
"placement": "right"
|
||||
},
|
||||
"pieType": "pie",
|
||||
"reduceOptions": {
|
||||
@@ -319,7 +328,7 @@
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showLegend": false,
|
||||
"showLegend": true,
|
||||
"strokeWidth": 1,
|
||||
"text": {}
|
||||
},
|
||||
@@ -335,6 +344,15 @@
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Value",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "renameByRegex",
|
||||
"options": {
|
||||
"regex": "(.*)",
|
||||
"renamePattern": "$1-how-much-wood-could-a-woodchuck-chuck-if-a-woodchuck-could-chuck-wood"
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -48,6 +48,23 @@ scopes:
|
||||
operator: equals
|
||||
value: kids
|
||||
|
||||
# This scope appears in multiple places in the tree.
|
||||
# The defaultPath determines which path is shown when this scope is selected
|
||||
# (e.g., from a URL or programmatically), even if another path also links to it.
|
||||
shared-service:
|
||||
title: Shared Service
|
||||
# Path from the root node down to the direct scopeNode.
|
||||
# Node names are hierarchical (parent-child), so use the full names.
|
||||
# This points to: gdev-scopes > production > shared-service-prod
|
||||
defaultPath:
|
||||
- gdev-scopes
|
||||
- gdev-scopes-production
|
||||
- gdev-scopes-production-shared-service-prod
|
||||
filters:
|
||||
- key: service
|
||||
operator: equals
|
||||
value: shared
|
||||
|
||||
tree:
|
||||
gdev-scopes:
|
||||
title: gdev-scopes
|
||||
@@ -68,6 +85,13 @@ tree:
|
||||
nodeType: leaf
|
||||
linkId: app2
|
||||
linkType: scope
|
||||
# This node links to 'shared-service' scope.
|
||||
# The scope's defaultPath points here (production > gdev-scopes).
|
||||
shared-service-prod:
|
||||
title: Shared Service
|
||||
nodeType: leaf
|
||||
linkId: shared-service
|
||||
linkType: scope
|
||||
test-cases:
|
||||
title: Test cases
|
||||
nodeType: container
|
||||
@@ -83,6 +107,15 @@ tree:
|
||||
nodeType: leaf
|
||||
linkId: test-case-2
|
||||
linkType: scope
|
||||
# This node also links to the same 'shared-service' scope.
|
||||
# However, the scope's defaultPath points to the production path,
|
||||
# so selecting this scope will expand the tree to production > shared-service-prod.
|
||||
shared-service-test:
|
||||
title: Shared Service (also in Production)
|
||||
subTitle: defaultPath points to Production
|
||||
nodeType: leaf
|
||||
linkId: shared-service
|
||||
linkType: scope
|
||||
test-case-redirect:
|
||||
title: Test case with redirect
|
||||
nodeType: leaf
|
||||
|
||||
+16
-4
@@ -51,8 +51,9 @@ type Config struct {
|
||||
|
||||
// ScopeConfig is used for YAML parsing - converts to v0alpha1.ScopeSpec
|
||||
type ScopeConfig struct {
|
||||
Title string `yaml:"title"`
|
||||
Filters []ScopeFilterConfig `yaml:"filters"`
|
||||
Title string `yaml:"title"`
|
||||
DefaultPath []string `yaml:"defaultPath,omitempty"`
|
||||
Filters []ScopeFilterConfig `yaml:"filters"`
|
||||
}
|
||||
|
||||
// ScopeFilterConfig is used for YAML parsing - converts to v0alpha1.ScopeFilter
|
||||
@@ -116,9 +117,20 @@ func convertScopeSpec(cfg ScopeConfig) v0alpha1.ScopeSpec {
|
||||
for i, f := range cfg.Filters {
|
||||
filters[i] = convertFilter(f)
|
||||
}
|
||||
|
||||
// Prefix defaultPath elements with the gdev prefix
|
||||
var defaultPath []string
|
||||
if len(cfg.DefaultPath) > 0 {
|
||||
defaultPath = make([]string, len(cfg.DefaultPath))
|
||||
for i, p := range cfg.DefaultPath {
|
||||
defaultPath[i] = prefix + "-" + p
|
||||
}
|
||||
}
|
||||
|
||||
return v0alpha1.ScopeSpec{
|
||||
Title: cfg.Title,
|
||||
Filters: filters,
|
||||
Title: cfg.Title,
|
||||
DefaultPath: defaultPath,
|
||||
Filters: filters,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ Plugin signature verification, also known as _signing_, is a security measure to
|
||||
|
||||
Learn more at [plugin policies](https://grafana.com/legal/plugins/).
|
||||
|
||||
## How does verifiction work?
|
||||
## How does verification work?
|
||||
|
||||
At startup, Grafana verifies the signatures of every plugin in the plugin directory.
|
||||
|
||||
|
||||
@@ -231,6 +231,10 @@ JSON Body schema:
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json; charset=UTF-8
|
||||
Content-Length: 644
|
||||
|
||||
```
|
||||
|
||||
The `metadata.continue` field contains a token to fetch the next page.
|
||||
|
||||
**Example subsequent request using continue token**:
|
||||
|
||||
@@ -521,6 +525,10 @@ JSON Body schema:
|
||||
```http
|
||||
HTTP/1.1 200
|
||||
Content-Type: application/json
|
||||
|
||||
```
|
||||
|
||||
Status Codes:
|
||||
|
||||
- **200** – Deleted
|
||||
- **401** – Unauthorized
|
||||
|
||||
@@ -259,6 +259,10 @@ JSON Body schema:
|
||||
|
||||
```
|
||||
|
||||
### Delete folder
|
||||
|
||||
`DELETE /apis/folder.grafana.app/v1beta1/namespaces/:namespace/folders/:uid`
|
||||
|
||||
Deletes an existing folder identified by UID along with all dashboards (and their alerts) stored in the folder. This operation cannot be reverted.
|
||||
|
||||
If [Grafana Alerting](ref:alerting) is enabled, you can set an optional query parameter `forceDeleteRules=false` so that requests will fail with 400 (Bad Request) error if the folder contains any Grafana alerts. However, if this parameter is set to `true` then it will delete any Grafana alerts under this folder.
|
||||
@@ -337,6 +341,10 @@ JSON Body schema:
|
||||
Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
|
||||
```
|
||||
|
||||
**Example Response**:
|
||||
|
||||
```http
|
||||
HTTP/1.1 200
|
||||
Content-Type: application/json
|
||||
|
||||
```
|
||||
|
||||
@@ -66,7 +66,6 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
|
||||
| `sharingDashboardImage` | Enables image sharing functionality for dashboards | Yes |
|
||||
| `tabularNumbers` | Use fixed-width numbers globally in the UI | |
|
||||
| `azureResourcePickerUpdates` | Enables the updated Azure Monitor resource picker | Yes |
|
||||
| `tempoSearchBackendMigration` | Run search queries through the tempo backend | |
|
||||
| `opentsdbBackendMigration` | Run queries through the data source backend | |
|
||||
|
||||
## Public preview feature toggles
|
||||
|
||||
@@ -51,6 +51,8 @@ test.describe('Dashboard keybindings with new layouts', { tag: ['@dashboards'] }
|
||||
|
||||
await expect(dashboardPage.getByGrafanaSelector(selectors.components.PanelInspector.Json.content)).toBeVisible();
|
||||
|
||||
// Press Escape to close tooltip on the close button
|
||||
await page.keyboard.press('Escape');
|
||||
// Press Escape to close inspector
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
|
||||
@@ -58,6 +58,8 @@ test.describe(
|
||||
|
||||
await expect(dashboardPage.getByGrafanaSelector(selectors.components.PanelInspector.Json.content)).toBeVisible();
|
||||
|
||||
// Press Escape to close tooltip on the close button
|
||||
await page.keyboard.press('Escape');
|
||||
// Press Escape to close inspector
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
|
||||
@@ -82,9 +82,9 @@ test.describe('Panels test: Table - Kitchen Sink', { tag: ['@panels', '@table']
|
||||
await expect(getCellHeight(page, 1, longTextColIdx)).resolves.toBeLessThan(100);
|
||||
|
||||
// click cell inspect, check that cell inspection pops open in the side as we'd expect.
|
||||
await loremIpsumCell.getByLabel('Inspect value').click();
|
||||
const loremIpsumText = await loremIpsumCell.textContent();
|
||||
expect(loremIpsumText).toBeDefined();
|
||||
await loremIpsumCell.getByLabel('Inspect value').click();
|
||||
await expect(page.getByRole('dialog').getByText(loremIpsumText!)).toBeVisible();
|
||||
});
|
||||
|
||||
|
||||
@@ -1021,11 +1021,6 @@
|
||||
"count": 2
|
||||
}
|
||||
},
|
||||
"public/app/core/actions/index.ts": {
|
||||
"no-barrel-files/no-barrel-files": {
|
||||
"count": 4
|
||||
}
|
||||
},
|
||||
"public/app/core/components/AccessControl/PermissionList.tsx": {
|
||||
"no-restricted-syntax": {
|
||||
"count": 1
|
||||
@@ -2610,11 +2605,6 @@
|
||||
"count": 2
|
||||
}
|
||||
},
|
||||
"public/app/features/explore/hooks/useStateSync/index.ts": {
|
||||
"no-barrel-files/no-barrel-files": {
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"public/app/features/explore/spec/helper/setup.tsx": {
|
||||
"@typescript-eslint/no-explicit-any": {
|
||||
"count": 1
|
||||
@@ -4030,11 +4020,6 @@
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"public/app/plugins/datasource/parca/webpack.config.ts": {
|
||||
"no-barrel-files/no-barrel-files": {
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"public/app/plugins/datasource/prometheus/configuration/AzureAuthSettings.tsx": {
|
||||
"no-restricted-syntax": {
|
||||
"count": 1
|
||||
@@ -4103,7 +4088,7 @@
|
||||
"count": 1
|
||||
},
|
||||
"@typescript-eslint/no-explicit-any": {
|
||||
"count": 2
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"public/app/plugins/datasource/tempo/resultTransformer.ts": {
|
||||
|
||||
@@ -585,6 +585,8 @@ module.exports = [
|
||||
// FIXME: Remove once all enterprise issues are fixed -
|
||||
// we don't have a suppressions file/approach for enterprise code yet
|
||||
...enterpriseIgnores,
|
||||
// Ignore decoupled plugin webpack configs
|
||||
'public/app/**/webpack.config.ts',
|
||||
],
|
||||
rules: {
|
||||
'no-barrel-files/no-barrel-files': 'error',
|
||||
|
||||
+6
@@ -246,6 +246,8 @@ const injectedRtkApi = api
|
||||
facetLimit: queryArg.facetLimit,
|
||||
tags: queryArg.tags,
|
||||
libraryPanel: queryArg.libraryPanel,
|
||||
panelType: queryArg.panelType,
|
||||
dataSourceType: queryArg.dataSourceType,
|
||||
permission: queryArg.permission,
|
||||
sort: queryArg.sort,
|
||||
limit: queryArg.limit,
|
||||
@@ -674,6 +676,10 @@ export type SearchDashboardsAndFoldersApiArg = {
|
||||
tags?: string[];
|
||||
/** find dashboards that reference a given libraryPanel */
|
||||
libraryPanel?: string;
|
||||
/** find dashboards using panels of a given plugin type */
|
||||
panelType?: string;
|
||||
/** find dashboards using datasources of a given plugin type */
|
||||
dataSourceType?: string;
|
||||
/** permission needed for the resource (view, edit, admin) */
|
||||
permission?: 'view' | 'edit' | 'admin';
|
||||
/** sortable field */
|
||||
|
||||
@@ -5340,6 +5340,7 @@ export type OrgUserDto = {
|
||||
};
|
||||
authLabels?: string[];
|
||||
avatarUrl?: string;
|
||||
created?: string;
|
||||
email?: string;
|
||||
isDisabled?: boolean;
|
||||
isExternallySynced?: boolean;
|
||||
@@ -6118,6 +6119,7 @@ export type ChangeUserPasswordCommand = {
|
||||
export type UserSearchHitDto = {
|
||||
authLabels?: string[];
|
||||
avatarUrl?: string;
|
||||
created?: string;
|
||||
email?: string;
|
||||
id?: number;
|
||||
isAdmin?: boolean;
|
||||
|
||||
+14
-10
@@ -657,10 +657,6 @@ export interface FeatureToggles {
|
||||
*/
|
||||
rolePickerDrawer?: boolean;
|
||||
/**
|
||||
* Enable unified storage search
|
||||
*/
|
||||
unifiedStorageSearch?: boolean;
|
||||
/**
|
||||
* Enable sprinkles on unified storage search
|
||||
*/
|
||||
unifiedStorageSearchSprinkles?: boolean;
|
||||
@@ -957,7 +953,8 @@ export interface FeatureToggles {
|
||||
*/
|
||||
alertingBulkActionsInUI?: boolean;
|
||||
/**
|
||||
* Registers AuthZ /apis endpoint
|
||||
* Deprecated: Use kubernetesAuthzCoreRolesApi, kubernetesAuthzRolesApi, and kubernetesAuthzRoleBindingsApi instead
|
||||
* @deprecated
|
||||
*/
|
||||
kubernetesAuthzApis?: boolean;
|
||||
/**
|
||||
@@ -973,6 +970,18 @@ export interface FeatureToggles {
|
||||
*/
|
||||
kubernetesAuthzZanzanaSync?: boolean;
|
||||
/**
|
||||
* Registers AuthZ Core Roles /apis endpoint
|
||||
*/
|
||||
kubernetesAuthzCoreRolesApi?: boolean;
|
||||
/**
|
||||
* Registers AuthZ Roles /apis endpoint
|
||||
*/
|
||||
kubernetesAuthzRolesApi?: boolean;
|
||||
/**
|
||||
* Registers AuthZ Role Bindings /apis endpoint
|
||||
*/
|
||||
kubernetesAuthzRoleBindingsApi?: boolean;
|
||||
/**
|
||||
* Enables create, delete, and update mutations for resources owned by IAM identity
|
||||
*/
|
||||
kubernetesAuthnMutation?: boolean;
|
||||
@@ -1124,11 +1133,6 @@ export interface FeatureToggles {
|
||||
*/
|
||||
pluginContainers?: boolean;
|
||||
/**
|
||||
* Run search queries through the tempo backend
|
||||
* @default false
|
||||
*/
|
||||
tempoSearchBackendMigration?: boolean;
|
||||
/**
|
||||
* Prioritize loading plugins from the CDN before other sources
|
||||
* @default false
|
||||
*/
|
||||
|
||||
@@ -52,6 +52,7 @@ export const availableIconsIndex = {
|
||||
bookmark: true,
|
||||
'book-open': true,
|
||||
'brackets-curly': true,
|
||||
brain: true,
|
||||
'browser-alt': true,
|
||||
bug: true,
|
||||
building: true,
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { css, cx } from '@emotion/css';
|
||||
import { FloatingFocusManager, useFloating } from '@floating-ui/react';
|
||||
import RcDrawer from '@rc-component/drawer';
|
||||
import { useDialog } from '@react-aria/dialog';
|
||||
import { FocusScope } from '@react-aria/focus';
|
||||
import { useOverlay } from '@react-aria/overlays';
|
||||
import { ReactNode, useCallback, useEffect, useState } from 'react';
|
||||
import { ReactNode, useCallback, useEffect, useId, useState } from 'react';
|
||||
import * as React from 'react';
|
||||
|
||||
import { GrafanaTheme2 } from '@grafana/data';
|
||||
@@ -81,17 +79,16 @@ export function Drawer({
|
||||
const styles = useStyles2(getStyles);
|
||||
const wrapperStyles = useStyles2(getWrapperStyles, size);
|
||||
const dragStyles = useStyles2(getDragStyles);
|
||||
const titleId = useId();
|
||||
|
||||
const overlayRef = React.useRef(null);
|
||||
const { dialogProps, titleProps } = useDialog({}, overlayRef);
|
||||
const { overlayProps } = useOverlay(
|
||||
{
|
||||
isDismissable: false,
|
||||
isOpen: true,
|
||||
onClose,
|
||||
const { context, refs } = useFloating({
|
||||
open: true,
|
||||
onOpenChange: (open) => {
|
||||
if (!open) {
|
||||
onClose?.();
|
||||
}
|
||||
},
|
||||
overlayRef
|
||||
);
|
||||
});
|
||||
|
||||
// Adds body class while open so the toolbar nav can hide some actions while drawer is open
|
||||
useBodyClassWhileOpen();
|
||||
@@ -117,6 +114,8 @@ export function Drawer({
|
||||
minWidth,
|
||||
},
|
||||
}}
|
||||
aria-label={typeof title === 'string' ? selectors.components.Drawer.General.title(title) : undefined}
|
||||
aria-labelledby={typeof title !== 'string' ? titleId : undefined}
|
||||
width={''}
|
||||
motion={{
|
||||
motionAppear: true,
|
||||
@@ -129,18 +128,8 @@ export function Drawer({
|
||||
motionName: styles.maskMotion,
|
||||
}}
|
||||
>
|
||||
<FocusScope restoreFocus contain autoFocus>
|
||||
<div
|
||||
aria-label={
|
||||
typeof title === 'string'
|
||||
? selectors.components.Drawer.General.title(title)
|
||||
: selectors.components.Drawer.General.title('no title')
|
||||
}
|
||||
className={styles.container}
|
||||
{...overlayProps}
|
||||
{...dialogProps}
|
||||
ref={overlayRef}
|
||||
>
|
||||
<FloatingFocusManager context={context} modal>
|
||||
<div className={styles.container} ref={refs.setFloating}>
|
||||
{/* eslint-disable-next-line jsx-a11y/no-static-element-interactions */}
|
||||
<div
|
||||
className={cx(dragStyles.dragHandleVertical, styles.resizer)}
|
||||
@@ -159,7 +148,7 @@ export function Drawer({
|
||||
</div>
|
||||
{typeof title === 'string' ? (
|
||||
<Stack direction="column">
|
||||
<Text element="h3" truncate {...titleProps}>
|
||||
<Text element="h3" truncate>
|
||||
{title}
|
||||
</Text>
|
||||
{subtitle && (
|
||||
@@ -169,13 +158,13 @@ export function Drawer({
|
||||
)}
|
||||
</Stack>
|
||||
) : (
|
||||
title
|
||||
<div id={titleId}>{title}</div>
|
||||
)}
|
||||
{tabs && <div className={styles.tabsWrapper}>{tabs}</div>}
|
||||
</div>
|
||||
{!scrollableContent ? content : <ScrollContainer showScrollIndicators>{content}</ScrollContainer>}
|
||||
</div>
|
||||
</FocusScope>
|
||||
</FloatingFocusManager>
|
||||
</RcDrawer>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -153,6 +153,10 @@ interface BaseProps<TableData extends object> {
|
||||
* Optional way to set how the table is sorted from the beginning. Must be memoized.
|
||||
*/
|
||||
initialSortBy?: Array<SortingRule<TableData>>;
|
||||
/**
|
||||
* Disable the ability to remove sorting on columns (none -> asc -> desc -> asc)
|
||||
*/
|
||||
disableSortRemove?: boolean;
|
||||
}
|
||||
|
||||
interface WithExpandableRow<TableData extends object> extends BaseProps<TableData> {
|
||||
@@ -191,6 +195,7 @@ export function InteractiveTable<TableData extends object>({
|
||||
showExpandAll = false,
|
||||
fetchData,
|
||||
initialSortBy = [],
|
||||
disableSortRemove,
|
||||
}: Props<TableData>) {
|
||||
const styles = useStyles2(getStyles);
|
||||
const tableColumns = useMemo(() => {
|
||||
@@ -222,6 +227,7 @@ export function InteractiveTable<TableData extends object>({
|
||||
disableMultiSort: true,
|
||||
// If fetchData is provided, we disable client-side sorting
|
||||
manualSortBy: Boolean(fetchData),
|
||||
disableSortRemove,
|
||||
getRowId,
|
||||
initialState: {
|
||||
hiddenColumns: [
|
||||
|
||||
@@ -26,4 +26,8 @@ export interface Column<TableData extends object> {
|
||||
* If the provided function returns `false` the column will be hidden.
|
||||
*/
|
||||
visible?: (data: TableData[]) => boolean;
|
||||
/**
|
||||
* Determines starting sort direction when the column header is clicked.
|
||||
*/
|
||||
sortDescFirst?: boolean;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ export function getColumns<K extends object>(
|
||||
disableSortBy: !Boolean(column.sortType),
|
||||
width: column.disableGrow ? 0 : undefined,
|
||||
visible: column.visible,
|
||||
...(column.sortDescFirst !== undefined && { sortDescFirst: column.sortDescFirst }),
|
||||
...(column.cell && { Cell: column.cell }),
|
||||
})),
|
||||
];
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { cx } from '@emotion/css';
|
||||
import { useDialog } from '@react-aria/dialog';
|
||||
import { FocusScope } from '@react-aria/focus';
|
||||
import { OverlayContainer, useOverlay } from '@react-aria/overlays';
|
||||
import { PropsWithChildren, useRef, type JSX } from 'react';
|
||||
import * as React from 'react';
|
||||
import { FloatingFocusManager, useDismiss, useFloating, useInteractions, useRole } from '@floating-ui/react';
|
||||
import { OverlayContainer } from '@react-aria/overlays';
|
||||
import { PropsWithChildren, ReactNode, useId, type JSX } from 'react';
|
||||
|
||||
import { t } from '@grafana/i18n';
|
||||
|
||||
@@ -66,23 +64,26 @@ export function Modal(props: PropsWithChildren<Props>) {
|
||||
trapFocus = true,
|
||||
} = props;
|
||||
const styles = useStyles2(getModalStyles);
|
||||
const titleId = useId();
|
||||
|
||||
const ref = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Handle interacting outside the dialog and pressing
|
||||
// the Escape key to close the modal.
|
||||
const { overlayProps, underlayProps } = useOverlay(
|
||||
{ isKeyboardDismissDisabled: !closeOnEscape, isOpen, onClose: onDismiss },
|
||||
ref
|
||||
);
|
||||
|
||||
// Get props for the dialog and its title
|
||||
const { dialogProps, titleProps } = useDialog(
|
||||
{
|
||||
'aria-label': ariaLabel,
|
||||
const { context, refs } = useFloating({
|
||||
open: isOpen,
|
||||
onOpenChange: (open) => {
|
||||
if (!open) {
|
||||
onDismiss?.();
|
||||
}
|
||||
},
|
||||
ref
|
||||
);
|
||||
});
|
||||
|
||||
const dismiss = useDismiss(context, {
|
||||
enabled: closeOnEscape,
|
||||
});
|
||||
|
||||
const role = useRole(context, {
|
||||
role: 'dialog',
|
||||
});
|
||||
|
||||
const { getFloatingProps } = useInteractions([dismiss, role]);
|
||||
|
||||
if (!isOpen) {
|
||||
return null;
|
||||
@@ -96,12 +97,17 @@ export function Modal(props: PropsWithChildren<Props>) {
|
||||
role="presentation"
|
||||
className={styles.modalBackdrop}
|
||||
onClick={onClickBackdrop || (closeOnBackdropClick ? onDismiss : undefined)}
|
||||
{...underlayProps}
|
||||
/>
|
||||
<FocusScope contain={trapFocus} autoFocus restoreFocus>
|
||||
<div className={cx(styles.modal, className)} ref={ref} {...overlayProps} {...dialogProps}>
|
||||
<FloatingFocusManager context={context} modal={trapFocus}>
|
||||
<div
|
||||
className={cx(styles.modal, className)}
|
||||
ref={refs.setFloating}
|
||||
aria-label={ariaLabel}
|
||||
aria-labelledby={typeof title === 'string' ? titleId : undefined}
|
||||
{...getFloatingProps()}
|
||||
>
|
||||
<div className={headerClass}>
|
||||
{typeof title === 'string' && <DefaultModalHeader {...props} title={title} id={titleProps.id} />}
|
||||
{typeof title === 'string' && <DefaultModalHeader {...props} title={title} id={titleId} />}
|
||||
{
|
||||
// FIXME: custom title components won't get an accessible title.
|
||||
// Do we really want to support them or shall we just limit this ModalTabsHeader?
|
||||
@@ -118,12 +124,12 @@ export function Modal(props: PropsWithChildren<Props>) {
|
||||
</div>
|
||||
<div className={cx(styles.modalContent, contentClassName)}>{children}</div>
|
||||
</div>
|
||||
</FocusScope>
|
||||
</FloatingFocusManager>
|
||||
</OverlayContainer>
|
||||
);
|
||||
}
|
||||
|
||||
function ModalButtonRow({ leftItems, children }: { leftItems?: React.ReactNode; children: React.ReactNode }) {
|
||||
function ModalButtonRow({ leftItems, children }: { leftItems?: ReactNode; children: ReactNode }) {
|
||||
const styles = useStyles2(getModalStyles);
|
||||
|
||||
if (leftItems) {
|
||||
|
||||
@@ -75,5 +75,3 @@ return (
|
||||
</Toggletip>
|
||||
);
|
||||
```
|
||||
|
||||
<ArgTypes of={Toggletip} />
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { Meta, StoryFn } from '@storybook/react';
|
||||
import { useState } from 'react';
|
||||
|
||||
import { Button } from '../Button/Button';
|
||||
import { Drawer } from '../Drawer/Drawer';
|
||||
import { Field } from '../Forms/Field';
|
||||
import { Input } from '../Input/Input';
|
||||
import { Modal } from '../Modal/Modal';
|
||||
import { ScrollContainer } from '../ScrollContainer/ScrollContainer';
|
||||
import mdx from '../Toggletip/Toggletip.mdx';
|
||||
|
||||
@@ -133,4 +138,86 @@ LongContent.parameters = {
|
||||
},
|
||||
};
|
||||
|
||||
export const InsideDrawer: StoryFn<typeof Toggletip> = () => {
|
||||
const [isDrawerOpen, setIsDrawerOpen] = useState(false);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button onClick={() => setIsDrawerOpen(true)}>Open Drawer</Button>
|
||||
{isDrawerOpen && (
|
||||
<Drawer title="Drawer with Toggletip" onClose={() => setIsDrawerOpen(false)}>
|
||||
<div>
|
||||
<p style={{ marginBottom: '16px' }}>This demonstrates using Toggletip inside a Drawer.</p>
|
||||
<Toggletip
|
||||
title="Interactive Form"
|
||||
content={
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
|
||||
<Field label="Name">
|
||||
<Input placeholder="Enter your name" />
|
||||
</Field>
|
||||
<Button variant="primary" size="sm">
|
||||
Submit
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
footer="Focus should work correctly within this Toggletip"
|
||||
placement="bottom-start"
|
||||
>
|
||||
<Button>Click to show Toggletip</Button>
|
||||
</Toggletip>
|
||||
</div>
|
||||
</Drawer>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
InsideDrawer.parameters = {
|
||||
controls: {
|
||||
hideNoControlsWarning: true,
|
||||
exclude: ['title', 'content', 'footer', 'children', 'placement', 'theme', 'closeButton', 'portalRoot'],
|
||||
},
|
||||
};
|
||||
|
||||
export const InsideModal: StoryFn<typeof Toggletip> = () => {
|
||||
const [isModalOpen, setIsModalOpen] = useState(false);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button onClick={() => setIsModalOpen(true)}>Open Modal</Button>
|
||||
<Modal title="Modal with Toggletip" isOpen={isModalOpen} onDismiss={() => setIsModalOpen(false)}>
|
||||
<div>
|
||||
<p style={{ marginBottom: '16px' }}>This demonstrates using Toggletip inside a Modal.</p>
|
||||
<Modal.ButtonRow>
|
||||
<Toggletip
|
||||
title="Interactive Form"
|
||||
content={
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: '8px' }}>
|
||||
<Field label="Name">
|
||||
<Input placeholder="Enter your name" />
|
||||
</Field>
|
||||
<Button variant="primary" size="sm">
|
||||
Submit
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
footer="Focus should work correctly within this Toggletip"
|
||||
placement="bottom-start"
|
||||
>
|
||||
<Button>Click to show Toggletip</Button>
|
||||
</Toggletip>
|
||||
</Modal.ButtonRow>
|
||||
</div>
|
||||
</Modal>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
InsideDrawer.parameters = {
|
||||
controls: {
|
||||
hideNoControlsWarning: true,
|
||||
exclude: ['title', 'content', 'footer', 'children', 'placement', 'theme', 'closeButton', 'portalRoot'],
|
||||
},
|
||||
};
|
||||
|
||||
export default meta;
|
||||
|
||||
@@ -23,7 +23,7 @@ export interface UsersIndicatorProps {
|
||||
* https://developers.grafana.com/ui/latest/index.html?path=/docs/iconography-usersindicator--docs
|
||||
*/
|
||||
export const UsersIndicator = ({ users, onClick, limit = 4 }: UsersIndicatorProps) => {
|
||||
const styles = useStyles2(getStyles);
|
||||
const styles = useStyles2(getStyles, limit);
|
||||
if (!users.length) {
|
||||
return null;
|
||||
}
|
||||
@@ -39,6 +39,9 @@ export const UsersIndicator = ({ users, onClick, limit = 4 }: UsersIndicatorProp
|
||||
className={styles.container}
|
||||
aria-label={t('grafana-ui.users-indicator.container-label', 'Users indicator container')}
|
||||
>
|
||||
{users.slice(0, limitReached ? limit : limit + 1).map((userView, idx, arr) => (
|
||||
<UserIcon key={userView.user.name} userView={userView} />
|
||||
))}
|
||||
{limitReached && (
|
||||
<UserIcon onClick={onClick} userView={{ user: { name: 'Extra users' } }} showTooltip={false}>
|
||||
{tooManyUsers
|
||||
@@ -47,26 +50,30 @@ export const UsersIndicator = ({ users, onClick, limit = 4 }: UsersIndicatorProp
|
||||
: `+${extraUsers}`}
|
||||
</UserIcon>
|
||||
)}
|
||||
{users
|
||||
.slice(0, limitReached ? limit : limit + 1)
|
||||
.reverse()
|
||||
.map((userView) => (
|
||||
<UserIcon key={userView.user.name} userView={userView} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const getStyles = (theme: GrafanaTheme2) => {
|
||||
const getStyles = (theme: GrafanaTheme2, limit: number) => {
|
||||
return {
|
||||
container: css({
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
flexDirection: 'row-reverse',
|
||||
marginLeft: theme.spacing(1),
|
||||
isolation: 'isolate',
|
||||
|
||||
'& > button': {
|
||||
marginLeft: theme.spacing(-1), // Overlay the elements a bit on top of each other
|
||||
|
||||
// Ensure overlaying user icons are stacked correctly with z-index on each element
|
||||
...Object.fromEntries(
|
||||
Array.from({ length: limit }).map((_, idx) => [
|
||||
`&:nth-of-type(${idx + 1})`,
|
||||
{
|
||||
zIndex: limit - idx,
|
||||
},
|
||||
])
|
||||
),
|
||||
},
|
||||
}),
|
||||
dots: css({
|
||||
|
||||
@@ -119,6 +119,7 @@ const getStyles = (theme: GrafanaTheme2) => ({
|
||||
table: css({
|
||||
width: '100%',
|
||||
'th:first-child': {
|
||||
width: '100%',
|
||||
borderBottom: `1px solid ${theme.colors.border.weak}`,
|
||||
},
|
||||
}),
|
||||
|
||||
@@ -134,6 +134,7 @@ const getStyles = (theme: GrafanaTheme2) => {
|
||||
label: 'LegendLabelCell',
|
||||
maxWidth: 0,
|
||||
width: '100%',
|
||||
minWidth: theme.spacing(16),
|
||||
}),
|
||||
labelCellInner: css({
|
||||
label: 'LegendLabelCellInner',
|
||||
|
||||
@@ -1,26 +1,55 @@
|
||||
package generic
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/registry/generic"
|
||||
"k8s.io/apiserver/pkg/registry/generic/registry"
|
||||
"k8s.io/apiserver/pkg/storage"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
)
|
||||
|
||||
// SelectableFieldsOptions allows customizing field selector behavior for a resource.
|
||||
type SelectableFieldsOptions struct {
|
||||
// GetAttrs returns labels and fields for the object.
|
||||
// If nil, the default GetAttrs is used which only exposes metadata.name.
|
||||
GetAttrs func(obj runtime.Object) (labels.Set, fields.Set, error)
|
||||
}
|
||||
|
||||
func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter) (*registry.Store, error) {
|
||||
return NewRegistryStoreWithSelectableFields(scheme, resourceInfo, optsGetter, SelectableFieldsOptions{})
|
||||
}
|
||||
|
||||
// NewRegistryStoreWithSelectableFields creates a registry store with custom selectable fields support.
|
||||
// Use this when you need to filter resources by custom fields like spec.connection.name.
|
||||
func NewRegistryStoreWithSelectableFields(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter, fieldOpts SelectableFieldsOptions) (*registry.Store, error) {
|
||||
gv := resourceInfo.GroupVersion()
|
||||
gv.Version = runtime.APIVersionInternal
|
||||
strategy := NewStrategy(scheme, gv)
|
||||
if resourceInfo.IsClusterScoped() {
|
||||
strategy = strategy.WithClusterScope()
|
||||
}
|
||||
|
||||
// Use custom GetAttrs if provided, otherwise use default
|
||||
var attrFunc storage.AttrFunc
|
||||
var predicateFunc func(label labels.Selector, field fields.Selector) storage.SelectionPredicate
|
||||
if fieldOpts.GetAttrs != nil {
|
||||
attrFunc = fieldOpts.GetAttrs
|
||||
// Pass nil predicateFunc to use default behavior with custom attrFunc
|
||||
predicateFunc = nil
|
||||
} else {
|
||||
attrFunc = GetAttrs
|
||||
predicateFunc = Matcher
|
||||
}
|
||||
|
||||
store := ®istry.Store{
|
||||
NewFunc: resourceInfo.NewFunc,
|
||||
NewListFunc: resourceInfo.NewListFunc,
|
||||
KeyRootFunc: KeyRootFunc(resourceInfo.GroupResource()),
|
||||
KeyFunc: NamespaceKeyFunc(resourceInfo.GroupResource()),
|
||||
PredicateFunc: Matcher,
|
||||
PredicateFunc: predicateFunc,
|
||||
DefaultQualifiedResource: resourceInfo.GroupResource(),
|
||||
SingularQualifiedResource: resourceInfo.SingularGroupResource(),
|
||||
TableConvertor: resourceInfo.TableConverter(),
|
||||
@@ -28,7 +57,7 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o
|
||||
UpdateStrategy: strategy,
|
||||
DeleteStrategy: strategy,
|
||||
}
|
||||
options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs}
|
||||
options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: attrFunc}
|
||||
if err := store.CompleteWithOptions(options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
## Build artifacts
|
||||
|
||||
Put the resulting tar in your `grafana` OSS path:
|
||||
```sh
|
||||
go -C grafana run ./pkg/build/cmd artifacts -a targz:enterprise:linux/amd64 --alpine-base=alpine:3.22 --tag-format='{{ .version }}-{{ .buildID }}-{{ .arch }}' --grafana-dir="${PWD}/grafana" --enterprise-dir="${PWD}/grafana-enterprise"
|
||||
```
|
||||
|
||||
Also build the e2e test runner:
|
||||
```sh
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ./e2e-runner ./e2e/
|
||||
```
|
||||
|
||||
And then `chmod +x ./e2e-runner`.
|
||||
|
||||
## Running tests
|
||||
|
||||
Reporting tests with Image Renderer:
|
||||
```sh
|
||||
go run ./pkg/build/e2e --suite=e2e/extensions/enterprise/smtp-suite --license=e2e/extensions/enterprise/license.jwt --image-renderer
|
||||
```
|
||||
@@ -138,6 +138,10 @@ func run(ctx context.Context, cmd *cli.Command) error {
|
||||
}
|
||||
|
||||
if code != 0 {
|
||||
if stdout, _ := c.Stdout(ctx); len(stdout) > 0 {
|
||||
log.Printf("e2e test suite stdout:\n%s", stdout)
|
||||
}
|
||||
|
||||
return fmt.Errorf("e2e tests failed with exit code %d", code)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,10 +8,10 @@ import (
|
||||
|
||||
func RunSuite(d *dagger.Client, svc *dagger.Service, src *dagger.Directory, cache *dagger.CacheVolume, suite, runnerFlags string) *dagger.Container {
|
||||
command := fmt.Sprintf(
|
||||
"./e2e-runner cypress --start-grafana=false --cypress-video"+
|
||||
"./e2e-runner cypress --browser=electron --start-grafana=false --cypress-video"+
|
||||
" --grafana-base-url http://grafana:3001 --suite %s %s", suite, runnerFlags)
|
||||
|
||||
return WithYarnCache(WithGrafanaFrontend(d.Container().From("cypress/included:13.1.0"), src), cache).
|
||||
return WithYarnCache(WithGrafanaFrontend(d.Container().From("cypress/included:14.3.2"), src), cache).
|
||||
WithWorkdir("/src").
|
||||
WithServiceBinding("grafana", svc).
|
||||
WithExec([]string{"yarn", "install", "--immutable"}).
|
||||
|
||||
@@ -99,13 +99,15 @@ func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOp
|
||||
}
|
||||
|
||||
if opts.StartImageRenderer {
|
||||
container = container.WithEnvVariable("START_IMAGE_RENDERER", "true").
|
||||
WithExec([]string{"apt-get", "update"}).
|
||||
WithExec([]string{"apt-get", "install", "-y", "ca-certificates"})
|
||||
imageRendererSvc := d.Container().From("grafana/grafana-image-renderer:" + opts.ImageRendererVersion).
|
||||
WithExposedPort(8081).
|
||||
AsService()
|
||||
|
||||
if opts.ImageRendererVersion != "" {
|
||||
container = container.WithEnvVariable("IMAGE_RENDERER_VERSION", opts.ImageRendererVersion)
|
||||
}
|
||||
container = container.WithServiceBinding("image-renderer", imageRendererSvc).
|
||||
WithExec([]string{"apt-get", "update"}).
|
||||
WithExec([]string{"apt-get", "install", "-y", "ca-certificates"}).
|
||||
WithEnvVariable("GF_RENDERING_CALLBACK_URL", "http://grafana:3001/").
|
||||
WithEnvVariable("GF_RENDERING_SERVER_URL", "http://image-renderer:8081/render")
|
||||
}
|
||||
|
||||
// We add all GF_ environment variables to allow for overriding Grafana configuration.
|
||||
|
||||
@@ -74,7 +74,7 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
return err
|
||||
}
|
||||
|
||||
grpcClient, err := newUnifiedClient(cfg, sqlStore, featureToggles)
|
||||
grpcClient, err := newUnifiedMigratorClient(cfg, sqlStore, featureToggles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -92,7 +92,7 @@ func ToUnifiedStorage(c utils.CommandLine, cfg *setting.Cfg, sqlStore db.DB) err
|
||||
return runInteractiveMigration(ctx, cfg, opts, dashboardAccess, grpcClient, start)
|
||||
}
|
||||
|
||||
func runNonInteractiveMigration(ctx context.Context, opts legacy.MigrateOptions, dashboardAccess legacy.MigrationDashboardAccessor, grpcClient resource.ResourceClient, start time.Time) error {
|
||||
func runNonInteractiveMigration(ctx context.Context, opts legacy.MigrateOptions, dashboardAccess legacy.MigrationDashboardAccessor, grpcClient resource.MigratorClient, start time.Time) error {
|
||||
migrator := migrations.ProvideUnifiedMigrator(dashboardAccess, grpcClient)
|
||||
|
||||
opts.WithHistory = true // always include history in non-interactive mode
|
||||
@@ -109,7 +109,7 @@ func runNonInteractiveMigration(ctx context.Context, opts legacy.MigrateOptions,
|
||||
return nil
|
||||
}
|
||||
|
||||
func runInteractiveMigration(ctx context.Context, cfg *setting.Cfg, opts legacy.MigrateOptions, dashboardAccess legacy.MigrationDashboardAccessor, grpcClient resource.ResourceClient, start time.Time) error {
|
||||
func runInteractiveMigration(ctx context.Context, cfg *setting.Cfg, opts legacy.MigrateOptions, dashboardAccess legacy.MigrationDashboardAccessor, grpcClient resource.MigratorClient, start time.Time) error {
|
||||
yes, err := promptYesNo(fmt.Sprintf("Count legacy resources for namespace: %s?", opts.Namespace))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -225,7 +225,7 @@ func promptYesNo(prompt string) (bool, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func newUnifiedClient(cfg *setting.Cfg, sqlStore db.DB, featureToggles featuremgmt.FeatureToggles) (resource.ResourceClient, error) {
|
||||
func newUnifiedMigratorClient(cfg *setting.Cfg, sqlStore db.DB, featureToggles featuremgmt.FeatureToggles) (resource.MigratorClient, error) {
|
||||
return unified.ProvideUnifiedStorageClient(&unified.Options{
|
||||
Cfg: cfg,
|
||||
Features: featureToggles,
|
||||
|
||||
@@ -10,6 +10,7 @@ const (
|
||||
SearchServerRing string = "search-server-ring"
|
||||
SearchServerDistributor string = "search-server-distributor"
|
||||
StorageServer string = "storage-server"
|
||||
SearchServer string = "search-server"
|
||||
ZanzanaServer string = "zanzana-server"
|
||||
InstrumentationServer string = "instrumentation-server"
|
||||
FrontendServer string = "frontend-server"
|
||||
@@ -21,6 +22,7 @@ var dependencyMap = map[string][]string{
|
||||
SearchServerRing: {InstrumentationServer, MemberlistKV},
|
||||
GrafanaAPIServer: {InstrumentationServer},
|
||||
StorageServer: {InstrumentationServer, SearchServerRing},
|
||||
SearchServer: {InstrumentationServer, SearchServerRing},
|
||||
ZanzanaServer: {InstrumentationServer},
|
||||
SearchServerDistributor: {InstrumentationServer, MemberlistKV, SearchServerRing},
|
||||
Core: {},
|
||||
|
||||
@@ -11,91 +11,59 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
_ resource.ResourceClient = (*directResourceClient)(nil)
|
||||
_ resource.StorageClient = (*DirectStorageClient)(nil)
|
||||
)
|
||||
|
||||
// The direct client passes requests directly to the server using the *same* context
|
||||
func NewDirectResourceClient(server resource.ResourceServer) resource.ResourceClient {
|
||||
return &directResourceClient{server}
|
||||
// NewDirectStorageClient creates a client that passes requests directly to the server using the *same* context
|
||||
func NewDirectStorageClient(server resource.ResourceServer) *DirectStorageClient {
|
||||
return &DirectStorageClient{server}
|
||||
}
|
||||
|
||||
type directResourceClient struct {
|
||||
type DirectStorageClient struct {
|
||||
server resource.ResourceServer
|
||||
}
|
||||
|
||||
// Create implements ResourceClient.
|
||||
func (d *directResourceClient) Create(ctx context.Context, in *resourcepb.CreateRequest, opts ...grpc.CallOption) (*resourcepb.CreateResponse, error) {
|
||||
func (d *DirectStorageClient) Create(ctx context.Context, in *resourcepb.CreateRequest, _ ...grpc.CallOption) (*resourcepb.CreateResponse, error) {
|
||||
return d.server.Create(ctx, in)
|
||||
}
|
||||
|
||||
// Delete implements ResourceClient.
|
||||
func (d *directResourceClient) Delete(ctx context.Context, in *resourcepb.DeleteRequest, opts ...grpc.CallOption) (*resourcepb.DeleteResponse, error) {
|
||||
func (d *DirectStorageClient) Delete(ctx context.Context, in *resourcepb.DeleteRequest, _ ...grpc.CallOption) (*resourcepb.DeleteResponse, error) {
|
||||
return d.server.Delete(ctx, in)
|
||||
}
|
||||
|
||||
// GetBlob implements ResourceClient.
|
||||
func (d *directResourceClient) GetBlob(ctx context.Context, in *resourcepb.GetBlobRequest, opts ...grpc.CallOption) (*resourcepb.GetBlobResponse, error) {
|
||||
func (d *DirectStorageClient) GetBlob(ctx context.Context, in *resourcepb.GetBlobRequest, _ ...grpc.CallOption) (*resourcepb.GetBlobResponse, error) {
|
||||
return d.server.GetBlob(ctx, in)
|
||||
}
|
||||
|
||||
// GetStats implements ResourceClient.
|
||||
func (d *directResourceClient) GetStats(ctx context.Context, in *resourcepb.ResourceStatsRequest, opts ...grpc.CallOption) (*resourcepb.ResourceStatsResponse, error) {
|
||||
return d.server.GetStats(ctx, in)
|
||||
}
|
||||
|
||||
// IsHealthy implements ResourceClient.
|
||||
func (d *directResourceClient) IsHealthy(ctx context.Context, in *resourcepb.HealthCheckRequest, opts ...grpc.CallOption) (*resourcepb.HealthCheckResponse, error) {
|
||||
func (d *DirectStorageClient) IsHealthy(ctx context.Context, in *resourcepb.HealthCheckRequest, _ ...grpc.CallOption) (*resourcepb.HealthCheckResponse, error) {
|
||||
return d.server.IsHealthy(ctx, in)
|
||||
}
|
||||
|
||||
// List implements ResourceClient.
|
||||
func (d *directResourceClient) List(ctx context.Context, in *resourcepb.ListRequest, opts ...grpc.CallOption) (*resourcepb.ListResponse, error) {
|
||||
func (d *DirectStorageClient) List(ctx context.Context, in *resourcepb.ListRequest, _ ...grpc.CallOption) (*resourcepb.ListResponse, error) {
|
||||
return d.server.List(ctx, in)
|
||||
}
|
||||
|
||||
func (d *directResourceClient) ListManagedObjects(ctx context.Context, in *resourcepb.ListManagedObjectsRequest, opts ...grpc.CallOption) (*resourcepb.ListManagedObjectsResponse, error) {
|
||||
return d.server.ListManagedObjects(ctx, in)
|
||||
}
|
||||
|
||||
func (d *directResourceClient) CountManagedObjects(ctx context.Context, in *resourcepb.CountManagedObjectsRequest, opts ...grpc.CallOption) (*resourcepb.CountManagedObjectsResponse, error) {
|
||||
return d.server.CountManagedObjects(ctx, in)
|
||||
}
|
||||
|
||||
// PutBlob implements ResourceClient.
|
||||
func (d *directResourceClient) PutBlob(ctx context.Context, in *resourcepb.PutBlobRequest, opts ...grpc.CallOption) (*resourcepb.PutBlobResponse, error) {
|
||||
func (d *DirectStorageClient) PutBlob(ctx context.Context, in *resourcepb.PutBlobRequest, _ ...grpc.CallOption) (*resourcepb.PutBlobResponse, error) {
|
||||
return d.server.PutBlob(ctx, in)
|
||||
}
|
||||
|
||||
// Read implements ResourceClient.
|
||||
func (d *directResourceClient) Read(ctx context.Context, in *resourcepb.ReadRequest, opts ...grpc.CallOption) (*resourcepb.ReadResponse, error) {
|
||||
func (d *DirectStorageClient) Read(ctx context.Context, in *resourcepb.ReadRequest, _ ...grpc.CallOption) (*resourcepb.ReadResponse, error) {
|
||||
return d.server.Read(ctx, in)
|
||||
}
|
||||
|
||||
// Search implements ResourceClient.
|
||||
func (d *directResourceClient) Search(ctx context.Context, in *resourcepb.ResourceSearchRequest, opts ...grpc.CallOption) (*resourcepb.ResourceSearchResponse, error) {
|
||||
return d.server.Search(ctx, in)
|
||||
}
|
||||
|
||||
// Update implements ResourceClient.
|
||||
func (d *directResourceClient) Update(ctx context.Context, in *resourcepb.UpdateRequest, opts ...grpc.CallOption) (*resourcepb.UpdateResponse, error) {
|
||||
func (d *DirectStorageClient) Update(ctx context.Context, in *resourcepb.UpdateRequest, _ ...grpc.CallOption) (*resourcepb.UpdateResponse, error) {
|
||||
return d.server.Update(ctx, in)
|
||||
}
|
||||
|
||||
// Watch implements ResourceClient.
|
||||
func (d *directResourceClient) Watch(ctx context.Context, in *resourcepb.WatchRequest, opts ...grpc.CallOption) (resourcepb.ResourceStore_WatchClient, error) {
|
||||
func (d *DirectStorageClient) Watch(_ context.Context, _ *resourcepb.WatchRequest, _ ...grpc.CallOption) (resourcepb.ResourceStore_WatchClient, error) {
|
||||
return nil, fmt.Errorf("watch not supported with direct resource client")
|
||||
}
|
||||
|
||||
// BulkProcess implements resource.ResourceClient.
|
||||
func (d *directResourceClient) BulkProcess(ctx context.Context, opts ...grpc.CallOption) (resourcepb.BulkStore_BulkProcessClient, error) {
|
||||
return nil, fmt.Errorf("BulkProcess not supported with direct resource client")
|
||||
}
|
||||
|
||||
// RebuildIndexes implements resource.ResourceClient.
|
||||
func (b *directResourceClient) RebuildIndexes(ctx context.Context, req *resourcepb.RebuildIndexesRequest, opts ...grpc.CallOption) (*resourcepb.RebuildIndexesResponse, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (b *directResourceClient) GetQuotaUsage(ctx context.Context, req *resourcepb.QuotaUsageRequest, opts ...grpc.CallOption) (*resourcepb.QuotaUsageResponse, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"k8s.io/apiserver/pkg/registry/rest"
|
||||
|
||||
"github.com/grafana/authlib/types"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
|
||||
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
|
||||
@@ -40,7 +41,7 @@ func (s *DashboardStorage) NewStore(dash utils.ResourceInfo, scheme *runtime.Sch
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := legacy.NewDirectResourceClient(server) // same context
|
||||
client := legacy.NewDirectStorageClient(server) // same context
|
||||
optsGetter := apistore.NewRESTOptionsGetterForClient(client, nil,
|
||||
defaultOpts.StorageConfig.Config, nil,
|
||||
)
|
||||
|
||||
@@ -142,6 +142,24 @@ func (s *SearchHandler) GetAPIRoutes(defs map[string]common.OpenAPIDefinition) *
|
||||
Schema: spec.StringProperty(),
|
||||
},
|
||||
},
|
||||
{
|
||||
ParameterProps: spec3.ParameterProps{
|
||||
Name: "panelType",
|
||||
In: "query",
|
||||
Description: "find dashboards using panels of a given plugin type",
|
||||
Required: false,
|
||||
Schema: spec.StringProperty(),
|
||||
},
|
||||
},
|
||||
{
|
||||
ParameterProps: spec3.ParameterProps{
|
||||
Name: "dataSourceType",
|
||||
In: "query",
|
||||
Description: "find dashboards using datasources of a given plugin type",
|
||||
Required: false,
|
||||
Schema: spec.StringProperty(),
|
||||
},
|
||||
},
|
||||
{
|
||||
ParameterProps: spec3.ParameterProps{
|
||||
Name: "permission",
|
||||
@@ -430,14 +448,11 @@ func convertHttpSearchRequestToResourceSearchRequest(queryParams url.Values, use
|
||||
}
|
||||
}
|
||||
|
||||
// The facet term fields
|
||||
// Apply facet terms
|
||||
if facets, ok := queryParams["facet"]; ok {
|
||||
if queryParams.Has("facetLimit") {
|
||||
if parsed, err := strconv.Atoi(queryParams.Get("facetLimit")); err == nil && parsed > 0 {
|
||||
facetLimit = parsed
|
||||
if facetLimit > 1000 {
|
||||
facetLimit = 1000
|
||||
}
|
||||
facetLimit = min(parsed, 1000)
|
||||
}
|
||||
}
|
||||
searchRequest.Facet = make(map[string]*resourcepb.ResourceSearchRequest_Facet)
|
||||
@@ -449,21 +464,35 @@ func convertHttpSearchRequestToResourceSearchRequest(queryParams url.Values, use
|
||||
}
|
||||
}
|
||||
|
||||
// The tags filter
|
||||
if tags, ok := queryParams["tag"]; ok {
|
||||
if v, ok := queryParams["tag"]; ok {
|
||||
searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{
|
||||
Key: "tags",
|
||||
Operator: "=",
|
||||
Values: tags,
|
||||
Values: v,
|
||||
})
|
||||
}
|
||||
|
||||
// The libraryPanel filter
|
||||
if libraryPanel, ok := queryParams["libraryPanel"]; ok {
|
||||
if v, ok := queryParams["panelType"]; ok {
|
||||
searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{
|
||||
Key: resource.SEARCH_FIELD_PREFIX + builders.DASHBOARD_PANEL_TYPES,
|
||||
Operator: "=",
|
||||
Values: v,
|
||||
})
|
||||
}
|
||||
|
||||
if v, ok := queryParams["dataSourceType"]; ok {
|
||||
searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{
|
||||
Key: resource.SEARCH_FIELD_PREFIX + builders.DASHBOARD_DS_TYPES,
|
||||
Operator: "=",
|
||||
Values: v,
|
||||
})
|
||||
}
|
||||
|
||||
if v, ok := queryParams["libraryPanel"]; ok {
|
||||
searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{
|
||||
Key: builders.DASHBOARD_LIBRARY_PANEL_REFERENCE,
|
||||
Operator: "=",
|
||||
Values: libraryPanel,
|
||||
Values: v,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/folder"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
@@ -74,7 +73,7 @@ func RegisterAPIService(cfg *setting.Cfg,
|
||||
acService accesscontrol.Service,
|
||||
accessClient authlib.AccessClient,
|
||||
registerer prometheus.Registerer,
|
||||
unified resource.ResourceClient,
|
||||
unified resourcepb.ResourceIndexClient,
|
||||
zanzanaClient zanzana.Client,
|
||||
) *FolderAPIBuilder {
|
||||
builder := &FolderAPIBuilder{
|
||||
@@ -93,7 +92,7 @@ func RegisterAPIService(cfg *setting.Cfg,
|
||||
return builder
|
||||
}
|
||||
|
||||
func NewAPIService(ac authlib.AccessClient, searcher resource.ResourceClient, features featuremgmt.FeatureToggles, zanzanaClient zanzana.Client, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *FolderAPIBuilder {
|
||||
func NewAPIService(ac authlib.AccessClient, searcher resourcepb.ResourceIndexClient, features featuremgmt.FeatureToggles, zanzanaClient zanzana.Client, resourcePermissionsSvc *dynamic.NamespaceableResourceInterface) *FolderAPIBuilder {
|
||||
return &FolderAPIBuilder{
|
||||
features: features,
|
||||
accessClient: ac,
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/open-feature/go-sdk/openfeature"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -209,8 +211,16 @@ func (b *IdentityAccessManagementAPIBuilder) GetGroupVersion() schema.GroupVersi
|
||||
}
|
||||
|
||||
func (b *IdentityAccessManagementAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
|
||||
//nolint:staticcheck // not yet migrated to OpenFeature
|
||||
if b.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAuthzApis) {
|
||||
client := openfeature.NewDefaultClient()
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancelFn()
|
||||
|
||||
// Check if any of the AuthZ APIs are enabled
|
||||
enableCoreRolesApi := client.Boolean(ctx, featuremgmt.FlagKubernetesAuthzCoreRolesApi, false, openfeature.TransactionContext(ctx))
|
||||
enableRolesApi := client.Boolean(ctx, featuremgmt.FlagKubernetesAuthzRolesApi, false, openfeature.TransactionContext(ctx))
|
||||
enableRoleBindingsApi := client.Boolean(ctx, featuremgmt.FlagKubernetesAuthzRoleBindingsApi, false, openfeature.TransactionContext(ctx))
|
||||
|
||||
if enableCoreRolesApi || enableRolesApi || enableRoleBindingsApi {
|
||||
if err := iamv0.AddAuthZKnownTypes(scheme); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -244,10 +254,16 @@ func (b *IdentityAccessManagementAPIBuilder) AllowedV0Alpha1Resources() []string
|
||||
func (b *IdentityAccessManagementAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error {
|
||||
storage := map[string]rest.Storage{}
|
||||
|
||||
client := openfeature.NewDefaultClient()
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancelFn()
|
||||
|
||||
//nolint:staticcheck // not yet migrated to OpenFeature
|
||||
enableZanzanaSync := b.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAuthzZanzanaSync)
|
||||
//nolint:staticcheck // not yet migrated to OpenFeature
|
||||
enableAuthzApis := b.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAuthzApis)
|
||||
|
||||
enableCoreRolesApi := client.Boolean(ctx, featuremgmt.FlagKubernetesAuthzCoreRolesApi, false, openfeature.TransactionContext(ctx))
|
||||
enableRolesApi := client.Boolean(ctx, featuremgmt.FlagKubernetesAuthzRolesApi, false, openfeature.TransactionContext(ctx))
|
||||
enableRoleBindingsApi := client.Boolean(ctx, featuremgmt.FlagKubernetesAuthzRoleBindingsApi, false, openfeature.TransactionContext(ctx))
|
||||
|
||||
// teams + users must have shorter names because they are often used as part of another name
|
||||
opts.StorageOptsRegister(iamv0.TeamResourceInfo.GroupResource(), apistore.StorageOptions{
|
||||
@@ -283,17 +299,21 @@ func (b *IdentityAccessManagementAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *ge
|
||||
return err
|
||||
}
|
||||
|
||||
if enableAuthzApis {
|
||||
if enableCoreRolesApi {
|
||||
// v0alpha1
|
||||
if err := b.UpdateCoreRolesAPIGroup(apiGroupInfo, opts, storage, enableZanzanaSync); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if enableRolesApi {
|
||||
// Role registration is delegated to the RoleApiInstaller
|
||||
if err := b.roleApiInstaller.RegisterStorage(apiGroupInfo, &opts, storage); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if enableRoleBindingsApi {
|
||||
if err := b.UpdateRoleBindingsAPIGroup(apiGroupInfo, opts, storage, enableZanzanaSync); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -722,7 +742,7 @@ func NewLocalStore(resourceInfo utils.ResourceInfo, scheme *runtime.Scheme, defa
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := resource.NewLocalResourceClient(server)
|
||||
client := resource.NewLocalResourceClient(server, nil)
|
||||
optsGetter := apistore.NewRESTOptionsGetterForClient(client, nil, defaultOpts.StorageConfig.Config, nil)
|
||||
|
||||
store, err := grafanaregistry.NewRegistryStore(scheme, resourceInfo, optsGetter)
|
||||
|
||||
@@ -71,6 +71,98 @@ func (_c *MockJobProgressRecorder_Complete_Call) RunAndReturn(run func(context.C
|
||||
return _c
|
||||
}
|
||||
|
||||
// HasDirPathFailedDeletion provides a mock function with given fields: folderPath
|
||||
func (_m *MockJobProgressRecorder) HasDirPathFailedDeletion(folderPath string) bool {
|
||||
ret := _m.Called(folderPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for HasDirPathFailedDeletion")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string) bool); ok {
|
||||
r0 = rf(folderPath)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockJobProgressRecorder_HasDirPathFailedDeletion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HasDirPathFailedDeletion'
|
||||
type MockJobProgressRecorder_HasDirPathFailedDeletion_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// HasDirPathFailedDeletion is a helper method to define mock.On call
|
||||
// - folderPath string
|
||||
func (_e *MockJobProgressRecorder_Expecter) HasDirPathFailedDeletion(folderPath interface{}) *MockJobProgressRecorder_HasDirPathFailedDeletion_Call {
|
||||
return &MockJobProgressRecorder_HasDirPathFailedDeletion_Call{Call: _e.mock.On("HasDirPathFailedDeletion", folderPath)}
|
||||
}
|
||||
|
||||
func (_c *MockJobProgressRecorder_HasDirPathFailedDeletion_Call) Run(run func(folderPath string)) *MockJobProgressRecorder_HasDirPathFailedDeletion_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockJobProgressRecorder_HasDirPathFailedDeletion_Call) Return(_a0 bool) *MockJobProgressRecorder_HasDirPathFailedDeletion_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockJobProgressRecorder_HasDirPathFailedDeletion_Call) RunAndReturn(run func(string) bool) *MockJobProgressRecorder_HasDirPathFailedDeletion_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// HasDirPathFailedCreation provides a mock function with given fields: path
|
||||
func (_m *MockJobProgressRecorder) HasDirPathFailedCreation(path string) bool {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for HasDirPathFailedCreation")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string) bool); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockJobProgressRecorder_HasDirPathFailedCreation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HasDirPathFailedCreation'
|
||||
type MockJobProgressRecorder_HasDirPathFailedCreation_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// HasDirPathFailedCreation is a helper method to define mock.On call
|
||||
// - path string
|
||||
func (_e *MockJobProgressRecorder_Expecter) HasDirPathFailedCreation(path interface{}) *MockJobProgressRecorder_HasDirPathFailedCreation_Call {
|
||||
return &MockJobProgressRecorder_HasDirPathFailedCreation_Call{Call: _e.mock.On("HasDirPathFailedCreation", path)}
|
||||
}
|
||||
|
||||
func (_c *MockJobProgressRecorder_HasDirPathFailedCreation_Call) Run(run func(path string)) *MockJobProgressRecorder_HasDirPathFailedCreation_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockJobProgressRecorder_HasDirPathFailedCreation_Call) Return(_a0 bool) *MockJobProgressRecorder_HasDirPathFailedCreation_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockJobProgressRecorder_HasDirPathFailedCreation_Call) RunAndReturn(run func(string) bool) *MockJobProgressRecorder_HasDirPathFailedCreation_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Record provides a mock function with given fields: ctx, result
|
||||
func (_m *MockJobProgressRecorder) Record(ctx context.Context, result JobResourceResult) {
|
||||
_m.Called(ctx, result)
|
||||
|
||||
@@ -2,6 +2,7 @@ package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -9,6 +10,8 @@ import (
|
||||
"github.com/grafana/grafana-app-sdk/logging"
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/safepath"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
)
|
||||
|
||||
// maybeNotifyProgress will only notify if a certain amount of time has passed
|
||||
@@ -58,6 +61,8 @@ type jobProgressRecorder struct {
|
||||
notifyImmediatelyFn ProgressFn
|
||||
maybeNotifyFn ProgressFn
|
||||
summaries map[string]*provisioning.JobResourceSummary
|
||||
failedCreations []string // Tracks folder paths that failed to be created
|
||||
failedDeletions []string // Tracks resource paths that failed to be deleted
|
||||
}
|
||||
|
||||
func newJobProgressRecorder(ProgressFn ProgressFn) JobProgressRecorder {
|
||||
@@ -84,10 +89,26 @@ func (r *jobProgressRecorder) Record(ctx context.Context, result JobResourceResu
|
||||
if result.Error != nil {
|
||||
shouldLogError = true
|
||||
logErr = result.Error
|
||||
if len(r.errors) < 20 {
|
||||
r.errors = append(r.errors, result.Error.Error())
|
||||
|
||||
// Don't count ignored actions as errors in error count or error list
|
||||
if result.Action != repository.FileActionIgnored {
|
||||
if len(r.errors) < 20 {
|
||||
r.errors = append(r.errors, result.Error.Error())
|
||||
}
|
||||
r.errorCount++
|
||||
}
|
||||
|
||||
// Automatically track failed operations based on error type and action
|
||||
// Check if this is a PathCreationError (folder creation failure)
|
||||
var pathErr *resources.PathCreationError
|
||||
if errors.As(result.Error, &pathErr) {
|
||||
r.failedCreations = append(r.failedCreations, pathErr.Path)
|
||||
}
|
||||
|
||||
// Track failed deletions, any deletion will stop the deletion of the parent folder (as it won't be empty)
|
||||
if result.Action == repository.FileActionDeleted {
|
||||
r.failedDeletions = append(r.failedDeletions, result.Path)
|
||||
}
|
||||
r.errorCount++
|
||||
}
|
||||
|
||||
r.updateSummary(result)
|
||||
@@ -112,6 +133,8 @@ func (r *jobProgressRecorder) ResetResults() {
|
||||
r.errorCount = 0
|
||||
r.errors = nil
|
||||
r.summaries = make(map[string]*provisioning.JobResourceSummary)
|
||||
r.failedCreations = nil
|
||||
r.failedDeletions = nil
|
||||
}
|
||||
|
||||
func (r *jobProgressRecorder) SetMessage(ctx context.Context, msg string) {
|
||||
@@ -309,3 +332,29 @@ func (r *jobProgressRecorder) Complete(ctx context.Context, err error) provision
|
||||
|
||||
return jobStatus
|
||||
}
|
||||
|
||||
// HasDirPathFailedCreation checks if a path is nested under any failed folder creation
|
||||
func (r *jobProgressRecorder) HasDirPathFailedCreation(path string) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
for _, failedCreation := range r.failedCreations {
|
||||
if safepath.InDir(path, failedCreation) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HasDirPathFailedDeletion checks if any resource deletions failed under a folder path
|
||||
func (r *jobProgressRecorder) HasDirPathFailedDeletion(folderPath string) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
for _, failedDeletion := range r.failedDeletions {
|
||||
if safepath.InDir(failedDeletion, folderPath) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -252,3 +253,221 @@ func TestJobProgressRecorderWarningOnlyNoErrors(t *testing.T) {
|
||||
require.NotNil(t, finalStatus.Warnings)
|
||||
assert.Len(t, finalStatus.Warnings, 1)
|
||||
}
|
||||
|
||||
func TestJobProgressRecorderFolderFailureTracking(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a progress recorder
|
||||
mockProgressFn := func(ctx context.Context, status provisioning.JobStatus) error {
|
||||
return nil
|
||||
}
|
||||
recorder := newJobProgressRecorder(mockProgressFn).(*jobProgressRecorder)
|
||||
|
||||
// Record a folder creation failure with PathCreationError
|
||||
pathErr := &resources.PathCreationError{
|
||||
Path: "folder1/",
|
||||
Err: assert.AnError,
|
||||
}
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder1/file.json",
|
||||
Action: repository.FileActionCreated,
|
||||
Error: pathErr,
|
||||
})
|
||||
|
||||
// Record another PathCreationError for a different folder
|
||||
pathErr2 := &resources.PathCreationError{
|
||||
Path: "folder2/subfolder/",
|
||||
Err: assert.AnError,
|
||||
}
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder2/subfolder/file.json",
|
||||
Action: repository.FileActionCreated,
|
||||
Error: pathErr2,
|
||||
})
|
||||
|
||||
// Record a deletion failure
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder3/file1.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
// Record another deletion failure
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder4/subfolder/file2.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
// Verify failed creations are tracked
|
||||
recorder.mu.RLock()
|
||||
assert.Len(t, recorder.failedCreations, 2)
|
||||
assert.Contains(t, recorder.failedCreations, "folder1/")
|
||||
assert.Contains(t, recorder.failedCreations, "folder2/subfolder/")
|
||||
|
||||
// Verify failed deletions are tracked
|
||||
assert.Len(t, recorder.failedDeletions, 2)
|
||||
assert.Contains(t, recorder.failedDeletions, "folder3/file1.json")
|
||||
assert.Contains(t, recorder.failedDeletions, "folder4/subfolder/file2.json")
|
||||
recorder.mu.RUnlock()
|
||||
}
|
||||
|
||||
func TestJobProgressRecorderHasDirPathFailedCreation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a progress recorder
|
||||
mockProgressFn := func(ctx context.Context, status provisioning.JobStatus) error {
|
||||
return nil
|
||||
}
|
||||
recorder := newJobProgressRecorder(mockProgressFn).(*jobProgressRecorder)
|
||||
|
||||
// Add failed creations via Record
|
||||
pathErr1 := &resources.PathCreationError{
|
||||
Path: "folder1/",
|
||||
Err: assert.AnError,
|
||||
}
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder1/file.json",
|
||||
Action: repository.FileActionCreated,
|
||||
Error: pathErr1,
|
||||
})
|
||||
|
||||
pathErr2 := &resources.PathCreationError{
|
||||
Path: "folder2/subfolder/",
|
||||
Err: assert.AnError,
|
||||
}
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder2/subfolder/file.json",
|
||||
Action: repository.FileActionCreated,
|
||||
Error: pathErr2,
|
||||
})
|
||||
|
||||
// Test nested paths
|
||||
assert.True(t, recorder.HasDirPathFailedCreation("folder1/file.json"))
|
||||
assert.True(t, recorder.HasDirPathFailedCreation("folder1/nested/file.json"))
|
||||
assert.True(t, recorder.HasDirPathFailedCreation("folder2/subfolder/file.json"))
|
||||
|
||||
// Test non-nested paths
|
||||
assert.False(t, recorder.HasDirPathFailedCreation("folder2/file2.json"))
|
||||
assert.False(t, recorder.HasDirPathFailedCreation("folder2/othersubfolder/inside.json"))
|
||||
assert.False(t, recorder.HasDirPathFailedCreation("other/file.json"))
|
||||
assert.False(t, recorder.HasDirPathFailedCreation("folder3/file.json"))
|
||||
assert.False(t, recorder.HasDirPathFailedCreation("file.json"))
|
||||
}
|
||||
|
||||
func TestJobProgressRecorderHasDirPathFailedDeletion(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a progress recorder
|
||||
mockProgressFn := func(ctx context.Context, status provisioning.JobStatus) error {
|
||||
return nil
|
||||
}
|
||||
recorder := newJobProgressRecorder(mockProgressFn).(*jobProgressRecorder)
|
||||
|
||||
// Add failed deletions via Record
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder1/file1.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder2/subfolder/file2.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder3/nested/deep/file3.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
// Test folder paths with failed deletions
|
||||
assert.True(t, recorder.HasDirPathFailedDeletion("folder1/"))
|
||||
assert.True(t, recorder.HasDirPathFailedDeletion("folder2/"))
|
||||
assert.True(t, recorder.HasDirPathFailedDeletion("folder2/subfolder/"))
|
||||
assert.True(t, recorder.HasDirPathFailedDeletion("folder3/"))
|
||||
assert.True(t, recorder.HasDirPathFailedDeletion("folder3/nested/"))
|
||||
assert.True(t, recorder.HasDirPathFailedDeletion("folder3/nested/deep/"))
|
||||
|
||||
// Test folder paths without failed deletions
|
||||
assert.False(t, recorder.HasDirPathFailedDeletion("other/"))
|
||||
assert.False(t, recorder.HasDirPathFailedDeletion("different/"))
|
||||
assert.False(t, recorder.HasDirPathFailedDeletion("folder2/othersubfolder/"))
|
||||
assert.False(t, recorder.HasDirPathFailedDeletion("folder2/subfolder/othersubfolder/"))
|
||||
assert.False(t, recorder.HasDirPathFailedDeletion("folder3/nested/anotherdeep/"))
|
||||
assert.False(t, recorder.HasDirPathFailedDeletion("folder3/nested/deep/insidedeep/"))
|
||||
}
|
||||
|
||||
func TestJobProgressRecorderResetResults(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a progress recorder
|
||||
mockProgressFn := func(ctx context.Context, status provisioning.JobStatus) error {
|
||||
return nil
|
||||
}
|
||||
recorder := newJobProgressRecorder(mockProgressFn).(*jobProgressRecorder)
|
||||
|
||||
// Add some data via Record
|
||||
pathErr := &resources.PathCreationError{
|
||||
Path: "folder1/",
|
||||
Err: assert.AnError,
|
||||
}
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder1/file.json",
|
||||
Action: repository.FileActionCreated,
|
||||
Error: pathErr,
|
||||
})
|
||||
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder2/file.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
// Verify data is stored
|
||||
recorder.mu.RLock()
|
||||
assert.Len(t, recorder.failedCreations, 1)
|
||||
assert.Len(t, recorder.failedDeletions, 1)
|
||||
recorder.mu.RUnlock()
|
||||
|
||||
// Reset results
|
||||
recorder.ResetResults()
|
||||
|
||||
// Verify data is cleared
|
||||
recorder.mu.RLock()
|
||||
assert.Nil(t, recorder.failedCreations)
|
||||
assert.Nil(t, recorder.failedDeletions)
|
||||
recorder.mu.RUnlock()
|
||||
}
|
||||
|
||||
func TestJobProgressRecorderIgnoredActionsDontCountAsErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a progress recorder
|
||||
mockProgressFn := func(ctx context.Context, status provisioning.JobStatus) error {
|
||||
return nil
|
||||
}
|
||||
recorder := newJobProgressRecorder(mockProgressFn).(*jobProgressRecorder)
|
||||
|
||||
// Record an ignored action with error
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder1/file1.json",
|
||||
Action: repository.FileActionIgnored,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
// Record a real error for comparison
|
||||
recorder.Record(ctx, JobResourceResult{
|
||||
Path: "folder2/file2.json",
|
||||
Action: repository.FileActionCreated,
|
||||
Error: assert.AnError,
|
||||
})
|
||||
|
||||
// Verify error count doesn't include ignored actions
|
||||
recorder.mu.RLock()
|
||||
assert.Equal(t, 1, recorder.errorCount, "ignored actions should not be counted as errors")
|
||||
assert.Len(t, recorder.errors, 1, "ignored action errors should not be in error list")
|
||||
recorder.mu.RUnlock()
|
||||
}
|
||||
|
||||
@@ -29,6 +29,10 @@ type JobProgressRecorder interface {
|
||||
StrictMaxErrors(maxErrors int)
|
||||
SetRefURLs(ctx context.Context, refURLs *provisioning.RepositoryURLs)
|
||||
Complete(ctx context.Context, err error) provisioning.JobStatus
|
||||
// HasDirPathFailedCreation checks if a path has any folder creations that failed
|
||||
HasDirPathFailedCreation(path string) bool
|
||||
// HasDirPathFailedDeletion checks if a folderPath has any folder deletions that failed
|
||||
HasDirPathFailedDeletion(folderPath string) bool
|
||||
}
|
||||
|
||||
// Worker is a worker that can process a job
|
||||
|
||||
@@ -75,11 +75,47 @@ func FullSync(
|
||||
return applyChanges(ctx, changes, clients, repositoryResources, progress, tracer, maxSyncWorkers, metrics)
|
||||
}
|
||||
|
||||
// shouldSkipChange checks if a change should be skipped based on previous failures on parent/child folders.
|
||||
// If there is a previous failure on the path, we don't need to process the change as it will fail anyway.
|
||||
func shouldSkipChange(ctx context.Context, change ResourceFileChange, progress jobs.JobProgressRecorder, tracer tracing.Tracer) bool {
|
||||
if change.Action != repository.FileActionDeleted && progress.HasDirPathFailedCreation(change.Path) {
|
||||
skipCtx, skipSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes.skip_nested_resource")
|
||||
skipSpan.SetAttributes(attribute.String("path", change.Path))
|
||||
progress.Record(skipCtx, jobs.JobResourceResult{
|
||||
Path: change.Path,
|
||||
Action: repository.FileActionIgnored,
|
||||
Warning: fmt.Errorf("resource was not processed because the parent folder could not be created"),
|
||||
})
|
||||
skipSpan.End()
|
||||
return true
|
||||
}
|
||||
|
||||
if change.Action == repository.FileActionDeleted && safepath.IsDir(change.Path) && progress.HasDirPathFailedDeletion(change.Path) {
|
||||
skipCtx, skipSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes.skip_folder_with_failed_deletions")
|
||||
skipSpan.SetAttributes(attribute.String("path", change.Path))
|
||||
progress.Record(skipCtx, jobs.JobResourceResult{
|
||||
Path: change.Path,
|
||||
Action: repository.FileActionIgnored,
|
||||
Group: resources.FolderKind.Group,
|
||||
Kind: resources.FolderKind.Kind,
|
||||
Warning: fmt.Errorf("folder was not processed because children resources in its path could not be deleted"),
|
||||
})
|
||||
skipSpan.End()
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func applyChange(ctx context.Context, change ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if shouldSkipChange(ctx, change, progress, tracer) {
|
||||
return
|
||||
}
|
||||
|
||||
if change.Action == repository.FileActionDeleted {
|
||||
deleteCtx, deleteSpan := tracer.Start(ctx, "provisioning.sync.full.apply_changes.delete")
|
||||
result := jobs.JobResourceResult{
|
||||
@@ -138,6 +174,7 @@ func applyChange(ctx context.Context, change ResourceFileChange, clients resourc
|
||||
ensureFolderSpan.RecordError(err)
|
||||
ensureFolderSpan.End()
|
||||
progress.Record(ctx, result)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -253,8 +290,6 @@ func applyChanges(ctx context.Context, changes []ResourceFileChange, clients res
|
||||
}
|
||||
|
||||
func applyFoldersSerially(ctx context.Context, folders []ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer) error {
|
||||
logger := logging.FromContext(ctx)
|
||||
|
||||
for _, folder := range folders {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
@@ -264,23 +299,9 @@ func applyFoldersSerially(ctx context.Context, folders []ResourceFileChange, cli
|
||||
return err
|
||||
}
|
||||
|
||||
folderCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
|
||||
applyChange(folderCtx, folder, clients, repositoryResources, progress, tracer)
|
||||
|
||||
if folderCtx.Err() == context.DeadlineExceeded {
|
||||
logger.Error("operation timed out after 15 seconds", "path", folder.Path, "action", folder.Action)
|
||||
|
||||
recordCtx, recordCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
progress.Record(recordCtx, jobs.JobResourceResult{
|
||||
Path: folder.Path,
|
||||
Action: folder.Action,
|
||||
Error: fmt.Errorf("operation timed out after 15 seconds"),
|
||||
})
|
||||
recordCancel()
|
||||
}
|
||||
|
||||
cancel()
|
||||
wrapWithTimeout(ctx, 15*time.Second, func(timeoutCtx context.Context) {
|
||||
applyChange(timeoutCtx, folder, clients, repositoryResources, progress, tracer)
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -318,7 +339,9 @@ loop:
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
applyChangeWithTimeout(ctx, change, clients, repositoryResources, progress, tracer, logger)
|
||||
wrapWithTimeout(ctx, 15*time.Second, func(timeoutCtx context.Context) {
|
||||
applyChange(timeoutCtx, change, clients, repositoryResources, progress, tracer)
|
||||
})
|
||||
}(change)
|
||||
}
|
||||
|
||||
@@ -331,21 +354,10 @@ loop:
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func applyChangeWithTimeout(ctx context.Context, change ResourceFileChange, clients resources.ResourceClients, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer, logger logging.Logger) {
|
||||
changeCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
// wrapWithTimeout wraps a function call with a timeout context
|
||||
func wrapWithTimeout(ctx context.Context, timeout time.Duration, fn func(context.Context)) {
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
applyChange(changeCtx, change, clients, repositoryResources, progress, tracer)
|
||||
|
||||
if changeCtx.Err() == context.DeadlineExceeded {
|
||||
logger.Error("operation timed out after 15 seconds", "path", change.Path, "action", change.Action)
|
||||
|
||||
recordCtx, recordCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
progress.Record(recordCtx, jobs.JobResourceResult{
|
||||
Path: change.Path,
|
||||
Action: change.Action,
|
||||
Error: fmt.Errorf("operation timed out after 15 seconds"),
|
||||
})
|
||||
recordCancel()
|
||||
}
|
||||
fn(timeoutCtx)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
dynamicfake "k8s.io/client-go/dynamic/fake"
|
||||
k8testing "k8s.io/client-go/testing"
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
)
|
||||
|
||||
/*
|
||||
TestFullSync_HierarchicalErrorHandling tests the hierarchical error handling behavior:
|
||||
|
||||
FOLDER CREATION FAILURES:
|
||||
- When a folder fails to be created with PathCreationError, all nested resources are skipped
|
||||
- Nested resources are recorded with FileActionIgnored and error "folder was not processed because children resources in its path could not be deleted"
|
||||
- Only the folder creation error counts toward error limits
|
||||
- Nested resource skips do NOT count toward error limits
|
||||
|
||||
FOLDER DELETION FAILURES:
|
||||
- When a file deletion fails, it's tracked in failedDeletions
|
||||
- When cleaning up folders, we check HasDirPathFailedDeletion()
|
||||
- If children failed to delete, folder deletion is skipped with FileActionIgnored
|
||||
- This prevents orphaning resources that still exist
|
||||
|
||||
DELETIONS NOT AFFECTED BY CREATION FAILURES:
|
||||
- If a folder creation fails, deletion operations for resources in that folder still proceed
|
||||
- This is because the resource might already exist from a previous sync
|
||||
- Only creations/updates/renames are affected by failed folder creation
|
||||
|
||||
AUTOMATIC TRACKING:
|
||||
- Record() automatically detects PathCreationError and adds to failedCreations
|
||||
- Record() automatically detects deletion failures and adds to failedDeletions
|
||||
- No manual calls to AddFailedCreation/AddFailedDeletion needed
|
||||
*/
|
||||
func TestFullSync_HierarchicalErrorHandling(t *testing.T) { // nolint:gocyclo
|
||||
tests := []struct {
|
||||
name string
|
||||
setupMocks func(*repository.MockRepository, *resources.MockRepositoryResources, *resources.MockResourceClients, *jobs.MockJobProgressRecorder, *dynamicfake.FakeDynamicClient)
|
||||
changes []ResourceFileChange
|
||||
description string
|
||||
expectError bool
|
||||
errorContains string
|
||||
}{
|
||||
{
|
||||
name: "folder creation fails, nested file skipped",
|
||||
description: "When folder1/ fails to create, folder1/file.json should be skipped with FileActionIgnored",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "folder1/file.json", Action: repository.FileActionCreated},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, _ *dynamicfake.FakeDynamicClient) {
|
||||
// First, check if nested under failed creation - not yet
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file.json").Return(false).Once()
|
||||
|
||||
// WriteResourceFromFile fails with PathCreationError for folder1/
|
||||
folderErr := &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "folder1/file.json", "").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
// File will be recorded with error, triggering automatic tracking of folder1/ failure
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file.json" && r.Error != nil && r.Action == repository.FileActionCreated
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "folder creation fails, multiple nested resources skipped",
|
||||
description: "When folder1/ fails to create, all nested resources (subfolder, files) are skipped",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "folder1/file1.json", Action: repository.FileActionCreated},
|
||||
{Path: "folder1/subfolder/file2.json", Action: repository.FileActionCreated},
|
||||
{Path: "folder1/file3.json", Action: repository.FileActionCreated},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, _ *dynamicfake.FakeDynamicClient) {
|
||||
// First file triggers folder creation failure
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file1.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "folder1/file1.json", "").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file1.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Subsequent files in same folder are skipped
|
||||
progress.On("HasDirPathFailedCreation", "folder1/subfolder/file2.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/subfolder/file2.json" &&
|
||||
r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil &&
|
||||
r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file3.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file3.json" &&
|
||||
r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil &&
|
||||
r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "file deletion failure tracked",
|
||||
description: "When a file deletion fails, it's automatically tracked in failedDeletions",
|
||||
changes: []ResourceFileChange{
|
||||
{
|
||||
Path: "folder1/file.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Existing: &provisioning.ResourceListItem{
|
||||
Name: "file1",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
},
|
||||
},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, dynamicClient *dynamicfake.FakeDynamicClient) {
|
||||
gvk := schema.GroupVersionKind{Group: "dashboard.grafana.app", Kind: "Dashboard", Version: "v1"}
|
||||
gvr := schema.GroupVersionResource{Group: "dashboard.grafana.app", Resource: "dashboards", Version: "v1"}
|
||||
|
||||
clients.On("ForResource", mock.Anything, mock.MatchedBy(func(gvr schema.GroupVersionResource) bool {
|
||||
return gvr.Group == "dashboard.grafana.app"
|
||||
})).Return(dynamicClient.Resource(gvr), gvk, nil)
|
||||
|
||||
// File deletion fails
|
||||
dynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("permission denied")
|
||||
})
|
||||
|
||||
// File deletion recorded with error, automatically tracked in failedDeletions
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file.json" &&
|
||||
r.Action == repository.FileActionDeleted &&
|
||||
r.Error != nil
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deletion proceeds despite creation failure",
|
||||
description: "When folder1/ fails to create, deletion of folder1/file2.json still proceeds (resource might exist from previous sync)",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "folder1/file1.json", Action: repository.FileActionCreated},
|
||||
{
|
||||
Path: "folder1/file2.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Existing: &provisioning.ResourceListItem{
|
||||
Name: "file2",
|
||||
Group: "dashboard.grafana.app",
|
||||
Resource: "dashboards",
|
||||
},
|
||||
},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, dynamicClient *dynamicfake.FakeDynamicClient) {
|
||||
// Creation fails
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file1.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "folder1/file1.json", "").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file1.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Deletion proceeds (NOT checking HasDirPathFailedCreation for deletions)
|
||||
// Note: deletion will fail because resource doesn't exist, but that's fine for this test
|
||||
gvk := schema.GroupVersionKind{Group: "dashboard.grafana.app", Kind: "Dashboard", Version: "v1"}
|
||||
gvr := schema.GroupVersionResource{Group: "dashboard.grafana.app", Resource: "dashboards", Version: "v1"}
|
||||
|
||||
clients.On("ForResource", mock.Anything, mock.MatchedBy(func(gvr schema.GroupVersionResource) bool {
|
||||
return gvr.Group == "dashboard.grafana.app"
|
||||
})).Return(dynamicClient.Resource(gvr), gvk, nil)
|
||||
|
||||
// Record deletion attempt (will have error since resource doesn't exist, but that's ok)
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file2.json" &&
|
||||
r.Action == repository.FileActionDeleted
|
||||
// Not checking r.Error because resource doesn't exist in fake client
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multi-level nesting - all skipped",
|
||||
description: "When level1/ fails, level1/level2/level3/file.json is also skipped",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "level1/file1.json", Action: repository.FileActionCreated},
|
||||
{Path: "level1/level2/file2.json", Action: repository.FileActionCreated},
|
||||
{Path: "level1/level2/level3/file3.json", Action: repository.FileActionCreated},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, _ *dynamicfake.FakeDynamicClient) {
|
||||
// First file triggers level1/ failure
|
||||
progress.On("HasDirPathFailedCreation", "level1/file1.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "level1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "level1/file1.json", "").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "level1/file1.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// All nested files are skipped
|
||||
for _, path := range []string{"level1/level2/file2.json", "level1/level2/level3/file3.json"} {
|
||||
progress.On("HasDirPathFailedCreation", path).Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == path && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mixed success and failure",
|
||||
description: "When success/ works and failure/ fails, only failure/* are skipped",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "success/file1.json", Action: repository.FileActionCreated},
|
||||
{Path: "failure/file2.json", Action: repository.FileActionCreated},
|
||||
{Path: "failure/nested/file3.json", Action: repository.FileActionCreated},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, _ *dynamicfake.FakeDynamicClient) {
|
||||
// Success path works
|
||||
progress.On("HasDirPathFailedCreation", "success/file1.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "success/file1.json", "").
|
||||
Return("resource1", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "success/file1.json" && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
// Failure path fails
|
||||
progress.On("HasDirPathFailedCreation", "failure/file2.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "failure/", Err: fmt.Errorf("disk full")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "failure/file2.json", "").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "failure/file2.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Nested file in failure path is skipped
|
||||
progress.On("HasDirPathFailedCreation", "failure/nested/file3.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "failure/nested/file3.json" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "folder creation fails with explicit folder in changes",
|
||||
description: "When folder1/ is explicitly in changes and fails to create, all nested resources (subfolders and files) are skipped",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "folder1/", Action: repository.FileActionCreated},
|
||||
{Path: "folder1/subfolder/", Action: repository.FileActionCreated},
|
||||
{Path: "folder1/file1.json", Action: repository.FileActionCreated},
|
||||
{Path: "folder1/subfolder/file2.json", Action: repository.FileActionCreated},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, _ *dynamicfake.FakeDynamicClient) {
|
||||
progress.On("HasDirPathFailedCreation", "folder1/").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "folder1/").Return("", folderErr).Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "folder1/subfolder/").Return(true).Once()
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file1.json").Return(true).Once()
|
||||
progress.On("HasDirPathFailedCreation", "folder1/subfolder/file2.json").Return(true).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/subfolder/" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file1.json" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/subfolder/file2.json" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "folder deletion prevented when child deletion fails",
|
||||
description: "When a file deletion fails, folder deletion is skipped with FileActionIgnored to prevent orphaning resources",
|
||||
changes: []ResourceFileChange{
|
||||
{
|
||||
Path: "folder1/file1.json",
|
||||
Action: repository.FileActionDeleted,
|
||||
Existing: &provisioning.ResourceListItem{Name: "file1", Group: "dashboard.grafana.app", Resource: "dashboards"},
|
||||
},
|
||||
{Path: "folder1/", Action: repository.FileActionDeleted, Existing: &provisioning.ResourceListItem{Name: "folder1", Group: "folder.grafana.app", Resource: "Folder"}},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, dynamicClient *dynamicfake.FakeDynamicClient) {
|
||||
gvk := schema.GroupVersionKind{Group: "dashboard.grafana.app", Kind: "Dashboard", Version: "v1"}
|
||||
gvr := schema.GroupVersionResource{Group: "dashboard.grafana.app", Resource: "dashboards", Version: "v1"}
|
||||
|
||||
clients.On("ForResource", mock.Anything, mock.MatchedBy(func(gvr schema.GroupVersionResource) bool {
|
||||
return gvr.Group == "dashboard.grafana.app"
|
||||
})).Return(dynamicClient.Resource(gvr), gvk, nil)
|
||||
|
||||
dynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("permission denied")
|
||||
})
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file1.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedDeletion", "folder1/").Return(true).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple folder deletion failures",
|
||||
description: "When multiple independent folders have child deletion failures, all folder deletions are skipped",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "folder1/file1.json", Action: repository.FileActionDeleted, Existing: &provisioning.ResourceListItem{Name: "file1", Group: "dashboard.grafana.app", Resource: "dashboards"}},
|
||||
{Path: "folder1/", Action: repository.FileActionDeleted},
|
||||
{Path: "folder2/file2.json", Action: repository.FileActionDeleted, Existing: &provisioning.ResourceListItem{Name: "file2", Group: "dashboard.grafana.app", Resource: "dashboards"}},
|
||||
{Path: "folder2/", Action: repository.FileActionDeleted},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, dynamicClient *dynamicfake.FakeDynamicClient) {
|
||||
gvk := schema.GroupVersionKind{Group: "dashboard.grafana.app", Kind: "Dashboard", Version: "v1"}
|
||||
gvr := schema.GroupVersionResource{Group: "dashboard.grafana.app", Resource: "dashboards", Version: "v1"}
|
||||
clients.On("ForResource", mock.Anything, mock.MatchedBy(func(gvr schema.GroupVersionResource) bool {
|
||||
return gvr.Group == "dashboard.grafana.app"
|
||||
})).Return(dynamicClient.Resource(gvr), gvk, nil)
|
||||
|
||||
dynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("permission denied")
|
||||
})
|
||||
|
||||
for _, path := range []string{"folder1/file1.json", "folder2/file2.json"} {
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == path && r.Error != nil
|
||||
})).Return().Once()
|
||||
}
|
||||
|
||||
progress.On("HasDirPathFailedDeletion", "folder1/").Return(true).Once()
|
||||
progress.On("HasDirPathFailedDeletion", "folder2/").Return(true).Once()
|
||||
|
||||
for _, path := range []string{"folder1/", "folder2/"} {
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == path && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested subfolder deletion failure",
|
||||
description: "When a file deletion fails in a nested subfolder, both the subfolder and parent folder deletions are skipped",
|
||||
changes: []ResourceFileChange{
|
||||
{Path: "parent/subfolder/file.json", Action: repository.FileActionDeleted, Existing: &provisioning.ResourceListItem{Name: "file1", Group: "dashboard.grafana.app", Resource: "dashboards"}},
|
||||
{Path: "parent/subfolder/", Action: repository.FileActionDeleted, Existing: &provisioning.ResourceListItem{Name: "subfolder", Group: "folder.grafana.app", Resource: "Folder"}},
|
||||
{Path: "parent/", Action: repository.FileActionDeleted, Existing: &provisioning.ResourceListItem{Name: "parent", Group: "folder.grafana.app", Resource: "Folder"}},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, dynamicClient *dynamicfake.FakeDynamicClient) {
|
||||
gvk := schema.GroupVersionKind{Group: "dashboard.grafana.app", Kind: "Dashboard", Version: "v1"}
|
||||
gvr := schema.GroupVersionResource{Group: "dashboard.grafana.app", Resource: "dashboards", Version: "v1"}
|
||||
clients.On("ForResource", mock.Anything, mock.MatchedBy(func(gvr schema.GroupVersionResource) bool {
|
||||
return gvr.Group == "dashboard.grafana.app"
|
||||
})).Return(dynamicClient.Resource(gvr), gvk, nil)
|
||||
|
||||
dynamicClient.PrependReactor("delete", "dashboards", func(action k8testing.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("permission denied")
|
||||
})
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "parent/subfolder/file.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedDeletion", "parent/subfolder/").Return(true).Once()
|
||||
progress.On("HasDirPathFailedDeletion", "parent/").Return(true).Once()
|
||||
|
||||
for _, path := range []string{"parent/subfolder/", "parent/"} {
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == path && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
scheme := runtime.NewScheme()
|
||||
dynamicClient := dynamicfake.NewSimpleDynamicClient(scheme)
|
||||
|
||||
repo := repository.NewMockRepository(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
clients := resources.NewMockResourceClients(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
compareFn := NewMockCompareFn(t)
|
||||
|
||||
repo.On("Config").Return(&provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "test-repo"},
|
||||
Spec: provisioning.RepositorySpec{Title: "Test Repo"},
|
||||
})
|
||||
|
||||
tt.setupMocks(repo, repoResources, clients, progress, dynamicClient)
|
||||
|
||||
compareFn.On("Execute", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, len(tt.changes)).Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
err := FullSync(context.Background(), repo, compareFn.Execute, clients, "ref", repoResources, progress, tracing.NewNoopTracerService(), 10, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
if tt.errorContains != "" {
|
||||
require.Contains(t, err.Error(), tt.errorContains)
|
||||
}
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
progress.AssertExpectations(t)
|
||||
repoResources.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -213,6 +213,10 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
return nil
|
||||
})
|
||||
|
||||
progress.On("HasDirPathFailedCreation", mock.MatchedBy(func(path string) bool {
|
||||
return path == "dashboards/one.json" || path == "dashboards/two.json" || path == "dashboards/three.json"
|
||||
})).Return(false).Maybe()
|
||||
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, mock.MatchedBy(func(path string) bool {
|
||||
return path == "dashboards/one.json" || path == "dashboards/two.json" || path == "dashboards/three.json"
|
||||
}), "").Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil).Maybe()
|
||||
@@ -235,6 +239,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/test.json").Return(false)
|
||||
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
|
||||
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
|
||||
@@ -259,6 +264,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/test.json").Return(false)
|
||||
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
|
||||
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write error"))
|
||||
@@ -285,6 +291,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/test.json").Return(false)
|
||||
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
|
||||
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
|
||||
@@ -309,6 +316,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/test.json").Return(false)
|
||||
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "").
|
||||
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write error"))
|
||||
@@ -335,6 +343,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "one/two/three/").Return(false)
|
||||
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "one/two/three/").Return("some-folder", nil)
|
||||
progress.On("Record", mock.Anything, jobs.JobResourceResult{
|
||||
@@ -357,6 +366,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "one/two/three/").Return(false)
|
||||
|
||||
repoResources.On(
|
||||
"EnsureFolderPathExist",
|
||||
@@ -581,6 +591,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedDeletion", "to-be-deleted/").Return(false)
|
||||
|
||||
scheme := runtime.NewScheme()
|
||||
require.NoError(t, metav1.AddMetaToScheme(scheme))
|
||||
@@ -640,6 +651,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedDeletion", "to-be-deleted/").Return(false)
|
||||
|
||||
scheme := runtime.NewScheme()
|
||||
require.NoError(t, metav1.AddMetaToScheme(scheme))
|
||||
@@ -695,6 +707,7 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
},
|
||||
setupMocks: func(repo *repository.MockRepository, repoResources *resources.MockRepositoryResources, clients *resources.MockResourceClients, progress *jobs.MockJobProgressRecorder, compareFn *MockCompareFn) {
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/slow.json").Return(false)
|
||||
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/slow.json", "").
|
||||
Run(func(args mock.Arguments) {
|
||||
@@ -708,19 +721,13 @@ func TestFullSync_ApplyChanges(t *testing.T) { //nolint:gocyclo
|
||||
}).
|
||||
Return("", schema.GroupVersionKind{}, context.DeadlineExceeded)
|
||||
|
||||
// applyChange records the error from WriteResourceFromFile
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
|
||||
return result.Action == repository.FileActionCreated &&
|
||||
result.Path == "dashboards/slow.json" &&
|
||||
result.Error != nil &&
|
||||
result.Error.Error() == "writing resource from file dashboards/slow.json: context deadline exceeded"
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
|
||||
return result.Action == repository.FileActionCreated &&
|
||||
result.Path == "dashboards/slow.json" &&
|
||||
result.Error != nil &&
|
||||
result.Error.Error() == "operation timed out after 15 seconds"
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef
|
||||
if len(affectedFolders) > 0 {
|
||||
cleanupStart := time.Now()
|
||||
span.AddEvent("checking if impacted folders should be deleted", trace.WithAttributes(attribute.Int("affected_folders", len(affectedFolders))))
|
||||
err := cleanupOrphanedFolders(ctx, repo, affectedFolders, repositoryResources, tracer)
|
||||
err := cleanupOrphanedFolders(ctx, repo, affectedFolders, repositoryResources, tracer, progress)
|
||||
metrics.RecordIncrementalSyncPhase(jobs.IncrementalSyncPhaseCleanup, time.Since(cleanupStart))
|
||||
if err != nil {
|
||||
return tracing.Error(span, fmt.Errorf("cleanup orphaned folders: %w", err))
|
||||
@@ -85,6 +85,20 @@ func applyIncrementalChanges(ctx context.Context, diff []repository.VersionedFil
|
||||
return nil, tracing.Error(span, err)
|
||||
}
|
||||
|
||||
// Check if this resource is nested under a failed folder creation
|
||||
// This only applies to creation/update/rename operations, not deletions
|
||||
if change.Action != repository.FileActionDeleted && progress.HasDirPathFailedCreation(change.Path) {
|
||||
// Skip this resource since its parent folder failed to be created
|
||||
skipCtx, skipSpan := tracer.Start(ctx, "provisioning.sync.incremental.skip_nested_resource")
|
||||
progress.Record(skipCtx, jobs.JobResourceResult{
|
||||
Path: change.Path,
|
||||
Action: repository.FileActionIgnored,
|
||||
Warning: fmt.Errorf("resource was not processed because the parent folder could not be created"),
|
||||
})
|
||||
skipSpan.End()
|
||||
continue
|
||||
}
|
||||
|
||||
if err := resources.IsPathSupported(change.Path); err != nil {
|
||||
ensureFolderCtx, ensureFolderSpan := tracer.Start(ctx, "provisioning.sync.incremental.ensure_folder_path_exist")
|
||||
// Maintain the safe segment for empty folders
|
||||
@@ -98,7 +112,15 @@ func applyIncrementalChanges(ctx context.Context, diff []repository.VersionedFil
|
||||
if err != nil {
|
||||
ensureFolderSpan.RecordError(err)
|
||||
ensureFolderSpan.End()
|
||||
return nil, tracing.Error(span, fmt.Errorf("unable to create empty file folder: %w", err))
|
||||
|
||||
progress.Record(ensureFolderCtx, jobs.JobResourceResult{
|
||||
Path: change.Path,
|
||||
Action: repository.FileActionIgnored,
|
||||
Group: resources.FolderKind.Group,
|
||||
Kind: resources.FolderKind.Kind,
|
||||
Error: err,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
progress.Record(ensureFolderCtx, jobs.JobResourceResult{
|
||||
@@ -185,6 +207,7 @@ func cleanupOrphanedFolders(
|
||||
affectedFolders map[string]string,
|
||||
repositoryResources resources.RepositoryResources,
|
||||
tracer tracing.Tracer,
|
||||
progress jobs.JobProgressRecorder,
|
||||
) error {
|
||||
ctx, span := tracer.Start(ctx, "provisioning.sync.incremental.cleanup_orphaned_folders")
|
||||
defer span.End()
|
||||
@@ -198,6 +221,12 @@ func cleanupOrphanedFolders(
|
||||
for path, folderName := range affectedFolders {
|
||||
span.SetAttributes(attribute.String("folder", folderName))
|
||||
|
||||
// Check if any resources under this folder failed to delete
|
||||
if progress.HasDirPathFailedDeletion(path) {
|
||||
span.AddEvent("skipping orphaned folder cleanup: a child resource in its path failed to be deleted")
|
||||
continue
|
||||
}
|
||||
|
||||
// if we can no longer find the folder in git, then we can delete it from grafana
|
||||
_, err := readerRepo.Read(ctx, path, "")
|
||||
if err != nil && (errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) {
|
||||
|
||||
@@ -0,0 +1,623 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
)
|
||||
|
||||
/*
|
||||
TestIncrementalSync_HierarchicalErrorHandling tests the hierarchical error handling behavior:
|
||||
|
||||
FOLDER CREATION FAILURES:
|
||||
- When EnsureFolderPathExist fails with PathCreationError, the path is tracked
|
||||
- Subsequent resources under that path are skipped with FileActionIgnored
|
||||
- Only the initial folder creation error counts toward error limits
|
||||
- WriteResourceFromFile can also return PathCreationError for implicit folder creation
|
||||
|
||||
FOLDER DELETION FAILURES (cleanupOrphanedFolders):
|
||||
- When RemoveResourceFromFile fails, path is tracked in failedDeletions
|
||||
- In cleanupOrphanedFolders, HasDirPathFailedDeletion() is checked before RemoveFolder
|
||||
- If children failed to delete, folder cleanup is skipped with a span event
|
||||
|
||||
DELETIONS NOT AFFECTED BY CREATION FAILURES:
|
||||
- HasDirPathFailedCreation is NOT checked for FileActionDeleted
|
||||
- Deletions proceed even if their parent folder failed to be created
|
||||
- This handles cleanup of resources that exist from previous syncs
|
||||
|
||||
RENAME OPERATIONS:
|
||||
- RenameResourceFile can return PathCreationError for the destination folder
|
||||
- Renames are affected by failed destination folder creation
|
||||
- Renames are NOT skipped due to source folder creation failures
|
||||
|
||||
AUTOMATIC TRACKING:
|
||||
- Record() automatically detects PathCreationError via errors.As() and adds to failedCreations
|
||||
- Record() automatically detects FileActionDeleted with error and adds to failedDeletions
|
||||
- No manual tracking calls needed
|
||||
*/
|
||||
func TestIncrementalSync_HierarchicalErrorHandling(t *testing.T) { // nolint:gocyclo
|
||||
tests := []struct {
|
||||
name string
|
||||
setupMocks func(*repository.MockVersioned, *resources.MockRepositoryResources, *jobs.MockJobProgressRecorder)
|
||||
changes []repository.VersionedFileChange
|
||||
previousRef string
|
||||
currentRef string
|
||||
description string
|
||||
expectError bool
|
||||
errorContains string
|
||||
}{
|
||||
{
|
||||
name: "folder creation fails, nested file skipped",
|
||||
description: "When unsupported/ fails to create via EnsureFolderPathExist, nested file is skipped",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "unsupported/file.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "unsupported/nested/file2.txt", Ref: "new-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// First file triggers folder creation which fails
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/file.txt").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "unsupported/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/").Return("", folderErr).Once()
|
||||
|
||||
// First file recorded with error (note: error is from folder creation, but recorded against file)
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "unsupported/file.txt" &&
|
||||
r.Action == repository.FileActionIgnored &&
|
||||
r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Second file is skipped because parent folder failed
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/nested/file2.txt").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "unsupported/nested/file2.txt" &&
|
||||
r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil &&
|
||||
r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "WriteResourceFromFile returns PathCreationError, nested resources skipped",
|
||||
description: "When WriteResourceFromFile implicitly creates a folder and fails, nested resources are skipped",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "folder1/file1.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "folder1/file2.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "folder1/nested/file3.json", Ref: "new-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// First file write fails with PathCreationError
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file1.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "folder1/file1.json", "new-ref").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
// First file recorded with error, automatically tracked
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file1.json" &&
|
||||
r.Action == repository.FileActionCreated &&
|
||||
r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Subsequent files are skipped
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file2.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file2.json" && r.Action == repository.FileActionIgnored && r.Warning != nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "folder1/nested/file3.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/nested/file3.json" && r.Action == repository.FileActionIgnored && r.Warning != nil
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "file deletion fails, folder cleanup skipped",
|
||||
description: "When RemoveResourceFromFile fails, cleanupOrphanedFolders skips folder removal",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionDeleted, Path: "dashboards/file1.json", PreviousRef: "old-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// File deletion fails (deletions don't check HasDirPathFailedCreation)
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/file1.json", "old-ref").
|
||||
Return("dashboard-1", "folder-uid", schema.GroupVersionKind{Kind: "Dashboard"}, fmt.Errorf("permission denied")).Once()
|
||||
|
||||
// Error recorded, automatically tracked in failedDeletions
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "dashboards/file1.json" &&
|
||||
r.Action == repository.FileActionDeleted &&
|
||||
r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// During cleanup, folder deletion is skipped
|
||||
progress.On("HasDirPathFailedDeletion", "dashboards/").Return(true).Once()
|
||||
|
||||
// Note: RemoveFolder should NOT be called (verified via AssertNotCalled in test)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deletion proceeds despite creation failure",
|
||||
description: "When folder1/ creation fails, deletion of folder1/old.json still proceeds",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "folder1/new.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionDeleted, Path: "folder1/old.json", PreviousRef: "old-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// Creation fails
|
||||
progress.On("HasDirPathFailedCreation", "folder1/new.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "folder1/new.json", "new-ref").
|
||||
Return("", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/new.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Deletion proceeds (NOT checking HasDirPathFailedCreation for deletions)
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "folder1/old.json", "old-ref").
|
||||
Return("old-resource", "", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/old.json" &&
|
||||
r.Action == repository.FileActionDeleted &&
|
||||
r.Error == nil // Deletion succeeds!
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multi-level nesting cascade",
|
||||
description: "When level1/ fails, level1/level2/level3/file.json is also skipped",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "level1/file.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "level1/level2/file.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "level1/level2/level3/file.txt", Ref: "new-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// First file triggers level1/ failure
|
||||
progress.On("HasDirPathFailedCreation", "level1/file.txt").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "level1/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "level1/").Return("", folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "level1/file.txt" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
|
||||
// All nested files are skipped
|
||||
for _, path := range []string{"level1/level2/file.txt", "level1/level2/level3/file.txt"} {
|
||||
progress.On("HasDirPathFailedCreation", path).Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == path && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mixed success and failure",
|
||||
description: "When success/ works and failure/ fails, only failure/* are skipped",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "success/file1.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "success/nested/file2.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "failure/file3.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "failure/nested/file4.txt", Ref: "new-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// Success path works
|
||||
progress.On("HasDirPathFailedCreation", "success/file1.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "success/file1.json", "new-ref").
|
||||
Return("resource-1", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "success/file1.json" && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "success/nested/file2.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "success/nested/file2.json", "new-ref").
|
||||
Return("resource-2", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "success/nested/file2.json" && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
// Failure path fails
|
||||
progress.On("HasDirPathFailedCreation", "failure/file3.txt").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "failure/", Err: fmt.Errorf("disk full")}
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "failure/").Return("", folderErr).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "failure/file3.txt" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
|
||||
// Nested file in failure path is skipped
|
||||
progress.On("HasDirPathFailedCreation", "failure/nested/file4.txt").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "failure/nested/file4.txt" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "rename with failed destination folder",
|
||||
description: "When RenameResourceFile fails with PathCreationError for destination, rename is skipped",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{
|
||||
Action: repository.FileActionRenamed,
|
||||
Path: "newfolder/file.json",
|
||||
PreviousPath: "oldfolder/file.json",
|
||||
Ref: "new-ref",
|
||||
PreviousRef: "old-ref",
|
||||
},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// Rename fails with PathCreationError for destination folder
|
||||
progress.On("HasDirPathFailedCreation", "newfolder/file.json").Return(false).Once()
|
||||
folderErr := &resources.PathCreationError{Path: "newfolder/", Err: fmt.Errorf("permission denied")}
|
||||
repoResources.On("RenameResourceFile", mock.Anything, "oldfolder/file.json", "old-ref", "newfolder/file.json", "new-ref").
|
||||
Return("", "", schema.GroupVersionKind{}, folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "newfolder/file.json" &&
|
||||
r.Action == repository.FileActionRenamed &&
|
||||
r.Error != nil
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "renamed file still checked, subsequent nested resources skipped",
|
||||
description: "After rename fails for folder1/file.json, other folder1/* files are skipped",
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
changes: []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionRenamed, Path: "folder1/file1.json", PreviousPath: "old/file1.json", Ref: "new-ref", PreviousRef: "old-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "folder1/file2.json", Ref: "new-ref"},
|
||||
},
|
||||
setupMocks: func(repo *repository.MockVersioned, repoResources *resources.MockRepositoryResources, progress *jobs.MockJobProgressRecorder) {
|
||||
// Rename is NOT skipped for creation failures (it's checking the destination path)
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file1.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file1.json" &&
|
||||
r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
// Second file also skipped
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file2.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file2.json" && r.Action == repository.FileActionIgnored && r.Warning != nil
|
||||
})).Return().Once()
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
runHierarchicalErrorHandlingTest(t, tt)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type compositeRepoForTest struct {
|
||||
*repository.MockVersioned
|
||||
*repository.MockReader
|
||||
}
|
||||
|
||||
func runHierarchicalErrorHandlingTest(t *testing.T, tt struct {
|
||||
name string
|
||||
setupMocks func(*repository.MockVersioned, *resources.MockRepositoryResources, *jobs.MockJobProgressRecorder)
|
||||
changes []repository.VersionedFileChange
|
||||
previousRef string
|
||||
currentRef string
|
||||
description string
|
||||
expectError bool
|
||||
errorContains string
|
||||
}) {
|
||||
var repo repository.Versioned
|
||||
mockVersioned := repository.NewMockVersioned(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
// For tests that need cleanup (folder deletion), use composite repo
|
||||
if tt.name == "file deletion fails, folder cleanup skipped" {
|
||||
mockReader := repository.NewMockReader(t)
|
||||
repo = &compositeRepoForTest{
|
||||
MockVersioned: mockVersioned,
|
||||
MockReader: mockReader,
|
||||
}
|
||||
} else {
|
||||
repo = mockVersioned
|
||||
}
|
||||
|
||||
mockVersioned.On("CompareFiles", mock.Anything, tt.previousRef, tt.currentRef).Return(tt.changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, len(tt.changes)).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
tt.setupMocks(mockVersioned, repoResources, progress)
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, tt.previousRef, tt.currentRef, repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
if tt.errorContains != "" {
|
||||
require.Contains(t, err.Error(), tt.errorContains)
|
||||
}
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
progress.AssertExpectations(t)
|
||||
repoResources.AssertExpectations(t)
|
||||
// For deletion tests, verify RemoveFolder was NOT called
|
||||
if tt.name == "file deletion fails, folder cleanup skipped" {
|
||||
repoResources.AssertNotCalled(t, "RemoveFolder", mock.Anything, mock.Anything)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIncrementalSync_HierarchicalErrorHandling_FailedFolderCreation tests nested resource skipping
|
||||
func TestIncrementalSync_HierarchicalErrorHandling_FailedFolderCreation(t *testing.T) {
|
||||
repo := repository.NewMockVersioned(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
changes := []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "unsupported/file.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "unsupported/subfolder/file2.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "unsupported/file3.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "other/file.json", Ref: "new-ref"},
|
||||
}
|
||||
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 4).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
folderErr := &resources.PathCreationError{Path: "unsupported/", Err: fmt.Errorf("permission denied")}
|
||||
// First check is before it fails.
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/file.txt").Return(false).Once()
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/").Return("", folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "unsupported/file.txt" && r.Action == repository.FileActionIgnored && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/subfolder/file2.txt").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "unsupported/subfolder/file2.txt" && r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/file3.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "unsupported/file3.json" && r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "other/file.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "other/file.json", "new-ref").
|
||||
Return("test-resource", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "other/file.json" && r.Action == repository.FileActionCreated && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
require.NoError(t, err)
|
||||
progress.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestIncrementalSync_HierarchicalErrorHandling_FailedFileDeletion tests folder cleanup prevention
|
||||
func TestIncrementalSync_HierarchicalErrorHandling_FailedFileDeletion(t *testing.T) {
|
||||
mockVersioned := repository.NewMockVersioned(t)
|
||||
mockReader := repository.NewMockReader(t)
|
||||
repo := &compositeRepoForTest{MockVersioned: mockVersioned, MockReader: mockReader}
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
changes := []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionDeleted, Path: "dashboards/file1.json", PreviousRef: "old-ref"},
|
||||
}
|
||||
|
||||
mockVersioned.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 1).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
// Deletions don't check HasDirPathFailedCreation, they go straight to removal
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/file1.json", "old-ref").
|
||||
Return("dashboard-1", "folder-uid", schema.GroupVersionKind{Kind: "Dashboard"}, fmt.Errorf("permission denied")).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "dashboards/file1.json" && r.Action == repository.FileActionDeleted &&
|
||||
r.Error != nil && r.Error.Error() == "removing resource from file dashboards/file1.json: permission denied"
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedDeletion", "dashboards/").Return(true).Once()
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
require.NoError(t, err)
|
||||
progress.AssertExpectations(t)
|
||||
repoResources.AssertNotCalled(t, "RemoveFolder", mock.Anything, mock.Anything)
|
||||
}
|
||||
|
||||
// TestIncrementalSync_HierarchicalErrorHandling_DeletionNotAffectedByCreationFailure tests deletions proceed despite creation failures
|
||||
func TestIncrementalSync_HierarchicalErrorHandling_DeletionNotAffectedByCreationFailure(t *testing.T) {
|
||||
repo := repository.NewMockVersioned(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
changes := []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "folder1/file.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionDeleted, Path: "folder1/old.json", PreviousRef: "old-ref"},
|
||||
}
|
||||
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 2).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
// Creation fails
|
||||
progress.On("HasDirPathFailedCreation", "folder1/file.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "folder1/file.json", "new-ref").
|
||||
Return("", schema.GroupVersionKind{}, &resources.PathCreationError{Path: "folder1/", Err: fmt.Errorf("permission denied")}).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/file.json" && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
// Deletion should NOT be skipped (not checking HasDirPathFailedCreation for deletions)
|
||||
// Deletions don't check HasDirPathFailedCreation, they go straight to removal
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "folder1/old.json", "old-ref").
|
||||
Return("old-resource", "", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "folder1/old.json" && r.Action == repository.FileActionDeleted && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
require.NoError(t, err)
|
||||
progress.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestIncrementalSync_HierarchicalErrorHandling_MultiLevelNesting tests multi-level cascade
|
||||
func TestIncrementalSync_HierarchicalErrorHandling_MultiLevelNesting(t *testing.T) {
|
||||
repo := repository.NewMockVersioned(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
changes := []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "level1/file.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "level1/level2/file.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "level1/level2/level3/file.txt", Ref: "new-ref"},
|
||||
}
|
||||
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 3).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
folderErr := &resources.PathCreationError{Path: "level1/", Err: fmt.Errorf("permission denied")}
|
||||
// First check is before it fails.
|
||||
progress.On("HasDirPathFailedCreation", "level1/file.txt").Return(false).Once()
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "level1/").Return("", folderErr).Once()
|
||||
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "level1/file.txt" && r.Action == repository.FileActionIgnored && r.Error != nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "level1/level2/file.txt").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "level1/level2/file.txt" && r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "level1/level2/level3/file.txt").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "level1/level2/level3/file.txt" && r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
require.NoError(t, err)
|
||||
progress.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestIncrementalSync_HierarchicalErrorHandling_MixedSuccessAndFailure tests partial failures
|
||||
func TestIncrementalSync_HierarchicalErrorHandling_MixedSuccessAndFailure(t *testing.T) {
|
||||
repo := repository.NewMockVersioned(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
changes := []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionCreated, Path: "success/file1.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "success/nested/file2.json", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "failure/file3.txt", Ref: "new-ref"},
|
||||
{Action: repository.FileActionCreated, Path: "failure/nested/file4.txt", Ref: "new-ref"},
|
||||
}
|
||||
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 4).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "success/file1.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "success/file1.json", "new-ref").
|
||||
Return("resource-1", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "success/file1.json" && r.Action == repository.FileActionCreated && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "success/nested/file2.json").Return(false).Once()
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "success/nested/file2.json", "new-ref").
|
||||
Return("resource-2", schema.GroupVersionKind{Kind: "Dashboard"}, nil).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "success/nested/file2.json" && r.Action == repository.FileActionCreated && r.Error == nil
|
||||
})).Return().Once()
|
||||
|
||||
folderErr := &resources.PathCreationError{Path: "failure/", Err: fmt.Errorf("disk full")}
|
||||
progress.On("HasDirPathFailedCreation", "failure/file3.txt").Return(false).Once()
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "failure/").Return("", folderErr).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "failure/file3.txt" && r.Action == repository.FileActionIgnored
|
||||
})).Return().Once()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "failure/nested/file4.txt").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "failure/nested/file4.txt" && r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
require.NoError(t, err)
|
||||
progress.AssertExpectations(t)
|
||||
repoResources.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// TestIncrementalSync_HierarchicalErrorHandling_RenameWithFailedFolderCreation tests rename operations affected by folder failures
|
||||
func TestIncrementalSync_HierarchicalErrorHandling_RenameWithFailedFolderCreation(t *testing.T) {
|
||||
repo := repository.NewMockVersioned(t)
|
||||
repoResources := resources.NewMockRepositoryResources(t)
|
||||
progress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
changes := []repository.VersionedFileChange{
|
||||
{Action: repository.FileActionRenamed, Path: "newfolder/file.json", PreviousPath: "oldfolder/file.json", Ref: "new-ref", PreviousRef: "old-ref"},
|
||||
}
|
||||
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 1).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
progress.On("TooManyErrors").Return(nil).Maybe()
|
||||
|
||||
progress.On("HasDirPathFailedCreation", "newfolder/file.json").Return(true).Once()
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(r jobs.JobResourceResult) bool {
|
||||
return r.Path == "newfolder/file.json" && r.Action == repository.FileActionIgnored &&
|
||||
r.Warning != nil && r.Warning.Error() == "resource was not processed because the parent folder could not be created"
|
||||
})).Return().Once()
|
||||
|
||||
err := IncrementalSync(context.Background(), repo, "old-ref", "new-ref", repoResources, progress, tracing.NewNoopTracerService(), jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
require.NoError(t, err)
|
||||
progress.AssertExpectations(t)
|
||||
}
|
||||
@@ -92,6 +92,10 @@ func TestIncrementalSync(t *testing.T) {
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation checks
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/test.json").Return(false)
|
||||
progress.On("HasDirPathFailedCreation", "alerts/alert.yaml").Return(false)
|
||||
|
||||
// Mock successful resource writes
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "new-ref").
|
||||
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
|
||||
@@ -127,6 +131,9 @@ func TestIncrementalSync(t *testing.T) {
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation check
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/path/file.txt").Return(false)
|
||||
|
||||
// Mock folder creation
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/path/").
|
||||
Return("test-folder", nil)
|
||||
@@ -161,6 +168,9 @@ func TestIncrementalSync(t *testing.T) {
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation check
|
||||
progress.On("HasDirPathFailedCreation", ".unsupported/path/file.txt").Return(false)
|
||||
|
||||
progress.On("Record", mock.Anything, jobs.JobResourceResult{
|
||||
Action: repository.FileActionIgnored,
|
||||
Path: ".unsupported/path/file.txt",
|
||||
@@ -222,6 +232,9 @@ func TestIncrementalSync(t *testing.T) {
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation check
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/new.json").Return(false)
|
||||
|
||||
// Mock resource rename
|
||||
repoResources.On("RenameResourceFile", mock.Anything, "dashboards/old.json", "old-ref", "dashboards/new.json", "new-ref").
|
||||
Return("renamed-dashboard", "", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
|
||||
@@ -254,6 +267,10 @@ func TestIncrementalSync(t *testing.T) {
|
||||
progress.On("SetTotal", mock.Anything, 1).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation check
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/ignored.json").Return(false)
|
||||
|
||||
progress.On("Record", mock.Anything, jobs.JobResourceResult{
|
||||
Action: repository.FileActionIgnored,
|
||||
Path: "dashboards/ignored.json",
|
||||
@@ -277,16 +294,28 @@ func TestIncrementalSync(t *testing.T) {
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 1).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation check
|
||||
progress.On("HasDirPathFailedCreation", "unsupported/path/file.txt").Return(false)
|
||||
|
||||
// Mock folder creation error
|
||||
repoResources.On("EnsureFolderPathExist", mock.Anything, "unsupported/path/").
|
||||
Return("", fmt.Errorf("failed to create folder"))
|
||||
|
||||
// Mock progress recording with error
|
||||
progress.On("Record", mock.Anything, mock.MatchedBy(func(result jobs.JobResourceResult) bool {
|
||||
return result.Action == repository.FileActionIgnored &&
|
||||
result.Path == "unsupported/path/file.txt" &&
|
||||
result.Error != nil &&
|
||||
result.Error.Error() == "failed to create folder"
|
||||
})).Return()
|
||||
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
},
|
||||
previousRef: "old-ref",
|
||||
currentRef: "new-ref",
|
||||
expectedError: "unable to create empty file folder: failed to create folder",
|
||||
expectedCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "error writing resource",
|
||||
@@ -303,6 +332,9 @@ func TestIncrementalSync(t *testing.T) {
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
|
||||
// Mock HasDirPathFailedCreation check
|
||||
progress.On("HasDirPathFailedCreation", "dashboards/test.json").Return(false)
|
||||
|
||||
// Mock resource write error
|
||||
repoResources.On("WriteResourceFromFile", mock.Anything, "dashboards/test.json", "new-ref").
|
||||
Return("test-dashboard", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, fmt.Errorf("write failed"))
|
||||
@@ -372,7 +404,8 @@ func TestIncrementalSync(t *testing.T) {
|
||||
repo.On("CompareFiles", mock.Anything, "old-ref", "new-ref").Return(changes, nil)
|
||||
progress.On("SetTotal", mock.Anything, 1).Return()
|
||||
progress.On("SetMessage", mock.Anything, "replicating versioned changes").Return()
|
||||
// Mock too many errors
|
||||
|
||||
// Mock too many errors - this is checked before processing files, so HasDirPathFailedCreation won't be called
|
||||
progress.On("TooManyErrors").Return(fmt.Errorf("too many errors occurred"))
|
||||
},
|
||||
previousRef: "old-ref",
|
||||
@@ -428,6 +461,9 @@ func TestIncrementalSync_CleanupOrphanedFolders(t *testing.T) {
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/old.json", "old-ref").
|
||||
Return("old-dashboard", "folder-uid", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
|
||||
|
||||
// Mock HasDirPathFailedDeletion check for cleanup
|
||||
progress.On("HasDirPathFailedDeletion", "dashboards/").Return(false)
|
||||
|
||||
// if the folder is not found in git, there should be a call to remove the folder from grafana
|
||||
repo.MockReader.On("Read", mock.Anything, "dashboards/", "").
|
||||
Return((*repository.FileInfo)(nil), repository.ErrFileNotFound)
|
||||
@@ -453,6 +489,10 @@ func TestIncrementalSync_CleanupOrphanedFolders(t *testing.T) {
|
||||
progress.On("SetMessage", mock.Anything, "versioned changes replicated").Return()
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "dashboards/old.json", "old-ref").
|
||||
Return("old-dashboard", "folder-uid", schema.GroupVersionKind{Kind: "Dashboard", Group: "dashboards"}, nil)
|
||||
|
||||
// Mock HasDirPathFailedDeletion check for cleanup
|
||||
progress.On("HasDirPathFailedDeletion", "dashboards/").Return(false)
|
||||
|
||||
// if the folder still exists in git, there should not be a call to delete it from grafana
|
||||
repo.MockReader.On("Read", mock.Anything, "dashboards/", "").
|
||||
Return(&repository.FileInfo{}, nil)
|
||||
@@ -485,6 +525,13 @@ func TestIncrementalSync_CleanupOrphanedFolders(t *testing.T) {
|
||||
repoResources.On("RemoveResourceFromFile", mock.Anything, "alerts/old-alert.yaml", "old-ref").
|
||||
Return("old-alert", "folder-uid-2", schema.GroupVersionKind{Kind: "Alert", Group: "alerts"}, nil)
|
||||
|
||||
progress.On("Record", mock.Anything, mock.Anything).Return()
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
|
||||
// Mock HasDirPathFailedDeletion checks for cleanup
|
||||
progress.On("HasDirPathFailedDeletion", "dashboards/").Return(false)
|
||||
progress.On("HasDirPathFailedDeletion", "alerts/").Return(false)
|
||||
|
||||
// both not found in git, both should be deleted
|
||||
repo.MockReader.On("Read", mock.Anything, "dashboards/", "").
|
||||
Return((*repository.FileInfo)(nil), repository.ErrFileNotFound)
|
||||
@@ -492,9 +539,6 @@ func TestIncrementalSync_CleanupOrphanedFolders(t *testing.T) {
|
||||
Return((*repository.FileInfo)(nil), repository.ErrFileNotFound)
|
||||
repoResources.On("RemoveFolder", mock.Anything, "folder-uid-1").Return(nil)
|
||||
repoResources.On("RemoveFolder", mock.Anything, "folder-uid-2").Return(nil)
|
||||
|
||||
progress.On("Record", mock.Anything, mock.Anything).Return()
|
||||
progress.On("TooManyErrors").Return(nil)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -559,6 +559,22 @@ func (b *APIBuilder) InstallSchema(scheme *runtime.Scheme) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Register custom field label conversion for Repository to enable field selectors like spec.connection.name
|
||||
err = scheme.AddFieldLabelConversionFunc(
|
||||
provisioning.SchemeGroupVersion.WithKind("Repository"),
|
||||
func(label, value string) (string, string, error) {
|
||||
switch label {
|
||||
case "metadata.name", "metadata.namespace", "spec.connection.name":
|
||||
return label, value, nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("field label not supported for Repository: %s", label)
|
||||
}
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metav1.AddToGroupVersion(scheme, provisioning.SchemeGroupVersion)
|
||||
// Only 1 version (for now?)
|
||||
return scheme.SetVersionPriority(provisioning.SchemeGroupVersion)
|
||||
@@ -569,10 +585,19 @@ func (b *APIBuilder) AllowedV0Alpha1Resources() []string {
|
||||
}
|
||||
|
||||
func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error {
|
||||
repositoryStorage, err := grafanaregistry.NewRegistryStore(opts.Scheme, provisioning.RepositoryResourceInfo, opts.OptsGetter)
|
||||
// Create repository storage with custom field selectors (e.g., spec.connection.name)
|
||||
repositoryStorage, err := grafanaregistry.NewRegistryStoreWithSelectableFields(
|
||||
opts.Scheme,
|
||||
provisioning.RepositoryResourceInfo,
|
||||
opts.OptsGetter,
|
||||
grafanaregistry.SelectableFieldsOptions{
|
||||
GetAttrs: RepositoryGetAttrs,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create repository storage: %w", err)
|
||||
}
|
||||
|
||||
repositoryStatusStorage := grafanaregistry.NewRegistryStatusStore(opts.Scheme, repositoryStorage)
|
||||
b.store = repositoryStorage
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package provisioning
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/registry/generic"
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
)
|
||||
|
||||
// RepositoryToSelectableFields returns a field set that can be used for field selectors.
|
||||
// This includes standard metadata fields plus custom fields like spec.connection.name.
|
||||
func RepositoryToSelectableFields(obj *provisioning.Repository) fields.Set {
|
||||
objectMetaFields := generic.ObjectMetaFieldsSet(&obj.ObjectMeta, true)
|
||||
|
||||
// Add custom selectable fields
|
||||
specificFields := fields.Set{
|
||||
"spec.connection.name": getConnectionName(obj),
|
||||
}
|
||||
|
||||
return generic.MergeFieldsSets(objectMetaFields, specificFields)
|
||||
}
|
||||
|
||||
// getConnectionName safely extracts the connection name from a Repository.
|
||||
// Returns empty string if no connection is configured.
|
||||
func getConnectionName(obj *provisioning.Repository) string {
|
||||
if obj == nil || obj.Spec.Connection == nil {
|
||||
return ""
|
||||
}
|
||||
return obj.Spec.Connection.Name
|
||||
}
|
||||
|
||||
// RepositoryGetAttrs returns labels and fields of a Repository object.
|
||||
// This is used by the storage layer for filtering.
|
||||
func RepositoryGetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) {
|
||||
repo, ok := obj.(*provisioning.Repository)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("given object is not a Repository")
|
||||
}
|
||||
return labels.Set(repo.Labels), RepositoryToSelectableFields(repo), nil
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package provisioning
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
)
|
||||
|
||||
func TestGetConnectionName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
repo *provisioning.Repository
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "nil repository returns empty string",
|
||||
repo: nil,
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "repository without connection returns empty string",
|
||||
repo: &provisioning.Repository{
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "test-repo",
|
||||
},
|
||||
},
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "repository with connection returns connection name",
|
||||
repo: &provisioning.Repository{
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "test-repo",
|
||||
Connection: &provisioning.ConnectionInfo{
|
||||
Name: "my-connection",
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: "my-connection",
|
||||
},
|
||||
{
|
||||
name: "repository with empty connection name returns empty string",
|
||||
repo: &provisioning.Repository{
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "test-repo",
|
||||
Connection: &provisioning.ConnectionInfo{
|
||||
Name: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := getConnectionName(tt.repo)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryToSelectableFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
repo *provisioning.Repository
|
||||
expectedFields map[string]string
|
||||
}{
|
||||
{
|
||||
name: "includes metadata.name and metadata.namespace",
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "default",
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "Test Repository",
|
||||
},
|
||||
},
|
||||
expectedFields: map[string]string{
|
||||
"metadata.name": "test-repo",
|
||||
"metadata.namespace": "default",
|
||||
"spec.connection.name": "",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "includes spec.connection.name when set",
|
||||
repo: &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "repo-with-connection",
|
||||
Namespace: "org-1",
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "Repo With Connection",
|
||||
Connection: &provisioning.ConnectionInfo{
|
||||
Name: "github-connection",
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedFields: map[string]string{
|
||||
"metadata.name": "repo-with-connection",
|
||||
"metadata.namespace": "org-1",
|
||||
"spec.connection.name": "github-connection",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fields := RepositoryToSelectableFields(tt.repo)
|
||||
|
||||
for key, expectedValue := range tt.expectedFields {
|
||||
actualValue, exists := fields[key]
|
||||
assert.True(t, exists, "field %s should exist", key)
|
||||
assert.Equal(t, expectedValue, actualValue, "field %s should have correct value", key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryGetAttrs(t *testing.T) {
|
||||
t.Run("returns error for non-Repository object", func(t *testing.T) {
|
||||
// Pass a different runtime.Object type instead of a Repository
|
||||
connection := &provisioning.Connection{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "not-a-repository",
|
||||
},
|
||||
}
|
||||
_, _, err := RepositoryGetAttrs(connection)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not a Repository")
|
||||
})
|
||||
|
||||
t.Run("returns labels and fields for valid Repository", func(t *testing.T) {
|
||||
repo := &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "default",
|
||||
Labels: map[string]string{
|
||||
"app": "grafana",
|
||||
"env": "test",
|
||||
},
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "Test Repository",
|
||||
Connection: &provisioning.ConnectionInfo{
|
||||
Name: "my-connection",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
labels, fields, err := RepositoryGetAttrs(repo)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check labels
|
||||
assert.Equal(t, "grafana", labels["app"])
|
||||
assert.Equal(t, "test", labels["env"])
|
||||
|
||||
// Check fields
|
||||
assert.Equal(t, "test-repo", fields["metadata.name"])
|
||||
assert.Equal(t, "default", fields["metadata.namespace"])
|
||||
assert.Equal(t, "my-connection", fields["spec.connection.name"])
|
||||
})
|
||||
|
||||
t.Run("returns empty connection name when not set", func(t *testing.T) {
|
||||
repo := &provisioning.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "default",
|
||||
},
|
||||
Spec: provisioning.RepositorySpec{
|
||||
Title: "Test Repository",
|
||||
},
|
||||
}
|
||||
|
||||
_, fields, err := RepositoryGetAttrs(repo)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", fields["spec.connection.name"])
|
||||
})
|
||||
}
|
||||
@@ -20,6 +20,21 @@ import (
|
||||
|
||||
const MaxNumberOfFolders = 10000
|
||||
|
||||
// PathCreationError represents an error that occurred while creating a folder path.
|
||||
// It contains the path that failed and the underlying error.
|
||||
type PathCreationError struct {
|
||||
Path string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *PathCreationError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
func (e *PathCreationError) Error() string {
|
||||
return fmt.Sprintf("failed to create path %s: %v", e.Path, e.Err)
|
||||
}
|
||||
|
||||
type FolderManager struct {
|
||||
repo repository.ReaderWriter
|
||||
tree FolderTree
|
||||
@@ -73,7 +88,11 @@ func (fm *FolderManager) EnsureFolderPathExist(ctx context.Context, filePath str
|
||||
}
|
||||
|
||||
if err := fm.EnsureFolderExists(ctx, f, parent); err != nil {
|
||||
return fmt.Errorf("ensure folder exists: %w", err)
|
||||
// Wrap in PathCreationError to indicate which path failed
|
||||
return &PathCreationError{
|
||||
Path: f.Path,
|
||||
Err: fmt.Errorf("ensure folder exists: %w", err),
|
||||
}
|
||||
}
|
||||
|
||||
fm.tree.Add(f, parent)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package resources_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPathCreationError(t *testing.T) {
|
||||
t.Run("Error method returns formatted message", func(t *testing.T) {
|
||||
underlyingErr := fmt.Errorf("underlying error")
|
||||
pathErr := &resources.PathCreationError{
|
||||
Path: "grafana/folder-1",
|
||||
Err: underlyingErr,
|
||||
}
|
||||
|
||||
expectedMsg := "failed to create path grafana/folder-1: underlying error"
|
||||
require.Equal(t, expectedMsg, pathErr.Error())
|
||||
})
|
||||
|
||||
t.Run("Unwrap returns underlying error", func(t *testing.T) {
|
||||
underlyingErr := fmt.Errorf("underlying error")
|
||||
pathErr := &resources.PathCreationError{
|
||||
Path: "grafana/folder-1",
|
||||
Err: underlyingErr,
|
||||
}
|
||||
|
||||
unwrapped := pathErr.Unwrap()
|
||||
require.Equal(t, underlyingErr, unwrapped)
|
||||
require.EqualError(t, unwrapped, "underlying error")
|
||||
})
|
||||
|
||||
t.Run("errors.Is finds underlying error", func(t *testing.T) {
|
||||
underlyingErr := fmt.Errorf("underlying error")
|
||||
pathErr := &resources.PathCreationError{
|
||||
Path: "grafana/folder-1",
|
||||
Err: underlyingErr,
|
||||
}
|
||||
|
||||
require.True(t, errors.Is(pathErr, underlyingErr))
|
||||
require.False(t, errors.Is(pathErr, fmt.Errorf("different error")))
|
||||
})
|
||||
|
||||
t.Run("errors.As extracts PathCreationError", func(t *testing.T) {
|
||||
underlyingErr := fmt.Errorf("underlying error")
|
||||
pathErr := &resources.PathCreationError{
|
||||
Path: "grafana/folder-1",
|
||||
Err: underlyingErr,
|
||||
}
|
||||
|
||||
var extractedErr *resources.PathCreationError
|
||||
require.True(t, errors.As(pathErr, &extractedErr))
|
||||
require.NotNil(t, extractedErr)
|
||||
require.Equal(t, "grafana/folder-1", extractedErr.Path)
|
||||
require.Equal(t, underlyingErr, extractedErr.Err)
|
||||
})
|
||||
|
||||
t.Run("errors.As returns false for non-PathCreationError", func(t *testing.T) {
|
||||
regularErr := fmt.Errorf("regular error")
|
||||
|
||||
var extractedErr *resources.PathCreationError
|
||||
require.False(t, errors.As(regularErr, &extractedErr))
|
||||
require.Nil(t, extractedErr)
|
||||
})
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
|
||||
alertingac "github.com/grafana/grafana/pkg/services/ngalert/accesscontrol"
|
||||
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/notifier/legacy_storage"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -25,7 +24,7 @@ var (
|
||||
)
|
||||
|
||||
type ReceiverService interface {
|
||||
GetReceiver(ctx context.Context, q ngmodels.GetReceiverQuery, user identity.Requester) (*ngmodels.Receiver, error)
|
||||
GetReceiver(ctx context.Context, uid string, decrypt bool, user identity.Requester) (*ngmodels.Receiver, error)
|
||||
GetReceivers(ctx context.Context, q ngmodels.GetReceiversQuery, user identity.Requester) ([]*ngmodels.Receiver, error)
|
||||
CreateReceiver(ctx context.Context, r *ngmodels.Receiver, orgID int64, user identity.Requester) (*ngmodels.Receiver, error)
|
||||
UpdateReceiver(ctx context.Context, r *ngmodels.Receiver, storedSecureFields map[string][]string, orgID int64, user identity.Requester) (*ngmodels.Receiver, error)
|
||||
@@ -116,22 +115,12 @@ func (s *legacyStorage) Get(ctx context.Context, uid string, _ *metav1.GetOption
|
||||
return nil, err
|
||||
}
|
||||
|
||||
name, err := legacy_storage.UidToName(uid)
|
||||
if err != nil {
|
||||
return nil, apierrors.NewNotFound(ResourceInfo.GroupResource(), uid)
|
||||
}
|
||||
q := ngmodels.GetReceiverQuery{
|
||||
OrgID: info.OrgID,
|
||||
Name: name,
|
||||
Decrypt: false,
|
||||
}
|
||||
|
||||
user, err := identity.GetRequester(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, err := s.service.GetReceiver(ctx, q, user)
|
||||
r, err := s.service.GetReceiver(ctx, uid, false, user)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -205,6 +205,14 @@ func (s *ModuleServer) Run() error {
|
||||
return sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, nil, s.log, s.registerer, docBuilders, s.storageMetrics, s.indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend)
|
||||
})
|
||||
|
||||
m.RegisterModule(modules.SearchServer, func() (services.Service, error) {
|
||||
docBuilders, err := InitializeDocumentBuilders(s.cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sql.ProvideUnifiedSearchGrpcService(s.cfg, s.features, nil, s.log, s.registerer, docBuilders, s.indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.storageBackend)
|
||||
})
|
||||
|
||||
m.RegisterModule(modules.ZanzanaServer, func() (services.Service, error) {
|
||||
return authz.ProvideZanzanaService(s.cfg, s.features, s.registerer)
|
||||
})
|
||||
|
||||
@@ -372,7 +372,7 @@ func initModuleServerForTest(
|
||||
return testModuleServer{server: ms, grpcAddress: cfg.GRPCServer.Address, httpPort: cfg.HTTPPort, healthClient: healthClient, id: cfg.InstanceID}
|
||||
}
|
||||
|
||||
func createBaselineServer(t *testing.T, dbType, dbConnStr string, testNamespaces []string) resource.ResourceServer {
|
||||
func createBaselineServer(t *testing.T, dbType, dbConnStr string, testNamespaces []string) resource.SearchServer {
|
||||
cfg := setting.NewCfg()
|
||||
section, err := cfg.Raw.NewSection("database")
|
||||
require.NoError(t, err)
|
||||
@@ -391,17 +391,20 @@ func createBaselineServer(t *testing.T, dbType, dbConnStr string, testNamespaces
|
||||
require.NoError(t, err)
|
||||
searchOpts, err := search.NewSearchOptions(features, cfg, docBuilders, nil, nil)
|
||||
require.NoError(t, err)
|
||||
server, err := sql.NewResourceServer(sql.ServerOptions{
|
||||
DB: nil,
|
||||
Cfg: cfg,
|
||||
Tracer: tracer,
|
||||
Reg: nil,
|
||||
AccessClient: nil,
|
||||
SearchOptions: searchOpts,
|
||||
StorageMetrics: nil,
|
||||
IndexMetrics: nil,
|
||||
Features: features,
|
||||
QOSQueue: nil,
|
||||
searchServer, err := sql.NewSearchServer(sql.SearchServerOptions{
|
||||
DB: nil,
|
||||
Cfg: cfg,
|
||||
Tracer: tracer,
|
||||
Reg: nil,
|
||||
AccessClient: nil,
|
||||
SearchOptions: searchOpts,
|
||||
IndexMetrics: nil,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
storageServer, err := sql.NewStorageServer(sql.StorageServerOptions{
|
||||
Cfg: cfg,
|
||||
Tracer: tracer,
|
||||
Features: features,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -417,12 +420,12 @@ func createBaselineServer(t *testing.T, dbType, dbConnStr string, testNamespaces
|
||||
|
||||
for _, ns := range testNamespaces {
|
||||
for range rand.Intn(maxPlaylistPerNamespace) + 1 {
|
||||
_, err = server.Create(ctx, generatePlaylistPayload(ns))
|
||||
_, err = storageServer.Create(ctx, generatePlaylistPayload(ns))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
return server
|
||||
return searchServer
|
||||
}
|
||||
|
||||
var counter int
|
||||
|
||||
Generated
+20
-3
@@ -513,6 +513,11 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storageMetrics := resource.ProvideStorageMetrics(registerer)
|
||||
storageBackend, err := sql.ProvideStorageBackend(cfg, sqlStore, tracer, registerer, storageMetrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options := &unified.Options{
|
||||
Cfg: cfg,
|
||||
Features: featureToggles,
|
||||
@@ -522,8 +527,8 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api
|
||||
Authzc: accessClient,
|
||||
Docs: documentBuilderSupplier,
|
||||
SecureValues: inlineSecureValueSupport,
|
||||
Backend: storageBackend,
|
||||
}
|
||||
storageMetrics := resource.ProvideStorageMetrics(registerer)
|
||||
bleveIndexMetrics := resource.ProvideIndexMetrics(registerer)
|
||||
resourceClient, err := unified.ProvideUnifiedStorageClient(options, storageMetrics, bleveIndexMetrics)
|
||||
if err != nil {
|
||||
@@ -1173,6 +1178,11 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storageMetrics := resource.ProvideStorageMetrics(registerer)
|
||||
storageBackend, err := sql.ProvideStorageBackend(cfg, sqlStore, tracer, registerer, storageMetrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options := &unified.Options{
|
||||
Cfg: cfg,
|
||||
Features: featureToggles,
|
||||
@@ -1182,8 +1192,8 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac
|
||||
Authzc: accessClient,
|
||||
Docs: documentBuilderSupplier,
|
||||
SecureValues: inlineSecureValueSupport,
|
||||
Backend: storageBackend,
|
||||
}
|
||||
storageMetrics := resource.ProvideStorageMetrics(registerer)
|
||||
bleveIndexMetrics := resource.ProvideIndexMetrics(registerer)
|
||||
resourceClient, err := unified.ProvideUnifiedStorageClient(options, storageMetrics, bleveIndexMetrics)
|
||||
if err != nil {
|
||||
@@ -1748,7 +1758,14 @@ func InitializeModuleServer(cfg *setting.Cfg, opts Options, apiOpts api.ServerOp
|
||||
hooksService := hooks.ProvideService()
|
||||
ossLicensingService := licensing.ProvideService(cfg, hooksService)
|
||||
moduleRegisterer := ProvideNoopModuleRegisterer()
|
||||
storageBackend, err := sql.ProvideStorageBackend(cfg)
|
||||
ossMigrations := migrations.ProvideOSSMigrations(featureToggles)
|
||||
inProcBus := bus.ProvideBus(tracingService)
|
||||
sqlStore, err := sqlstore.ProvideService(cfg, featureToggles, ossMigrations, inProcBus, tracingService)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tracer := otelTracer()
|
||||
storageBackend, err := sql.ProvideStorageBackend(cfg, sqlStore, tracer, registerer, storageMetrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,9 @@ package server
|
||||
|
||||
import (
|
||||
"github.com/google/wire"
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
|
||||
"github.com/grafana/grafana/pkg/configprovider"
|
||||
"github.com/grafana/grafana/pkg/infra/metrics"
|
||||
@@ -65,6 +68,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/storage/legacysql"
|
||||
"github.com/grafana/grafana/pkg/storage/unified"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
search2 "github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/search/builders"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql"
|
||||
@@ -145,8 +149,10 @@ var wireExtsBasicSet = wire.NewSet(
|
||||
sandbox.ProvideService,
|
||||
wire.Bind(new(sandbox.Sandbox), new(*sandbox.Service)),
|
||||
wire.Struct(new(unified.Options), "*"),
|
||||
unified.ProvideUnifiedStorageClient,
|
||||
sql.ProvideStorageBackend,
|
||||
unified.ProvideUnifiedStorageClient,
|
||||
wire.Bind(new(resourcepb.ResourceIndexClient), new(resource.ResourceClient)),
|
||||
wire.Bind(new(resource.MigratorClient), new(resource.ResourceClient)),
|
||||
builder.ProvideDefaultBuildHandlerChainFuncFromBuilders,
|
||||
aggregatorrunner.ProvideNoopAggregatorConfigurator,
|
||||
apisregistry.WireSetExts,
|
||||
@@ -195,6 +201,16 @@ var wireExtsModuleServerSet = wire.NewSet(
|
||||
tracing.ProvideTracingConfig,
|
||||
tracing.ProvideService,
|
||||
wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)),
|
||||
otelTracer,
|
||||
// Bus
|
||||
bus.ProvideBus,
|
||||
wire.Bind(new(bus.Bus), new(*bus.InProcBus)),
|
||||
// Database migrations
|
||||
migrations.ProvideOSSMigrations,
|
||||
wire.Bind(new(registry.DatabaseMigrator), new(*migrations.OSSMigrations)),
|
||||
// Database
|
||||
sqlstore.ProvideService,
|
||||
wire.Bind(new(db.DB), new(*sqlstore.SQLStore)),
|
||||
// Unified storage
|
||||
resource.ProvideStorageMetrics,
|
||||
resource.ProvideIndexMetrics,
|
||||
|
||||
@@ -1087,13 +1087,6 @@ var (
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: identityAccessTeam,
|
||||
},
|
||||
{
|
||||
Name: "unifiedStorageSearch",
|
||||
Description: "Enable unified storage search",
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: grafanaSearchAndStorageSquad,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
{
|
||||
Name: "unifiedStorageSearchSprinkles",
|
||||
Description: "Enable sprinkles on unified storage search",
|
||||
@@ -1585,8 +1578,8 @@ var (
|
||||
},
|
||||
{
|
||||
Name: "kubernetesAuthzApis",
|
||||
Description: "Registers AuthZ /apis endpoint",
|
||||
Stage: FeatureStageExperimental,
|
||||
Description: "Deprecated: Use kubernetesAuthzCoreRolesApi, kubernetesAuthzRolesApi, and kubernetesAuthzRoleBindingsApi instead",
|
||||
Stage: FeatureStageDeprecated,
|
||||
Owner: identityAccessTeam,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
@@ -1611,6 +1604,27 @@ var (
|
||||
Owner: identityAccessTeam,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
{
|
||||
Name: "kubernetesAuthzCoreRolesApi",
|
||||
Description: "Registers AuthZ Core Roles /apis endpoint",
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: identityAccessTeam,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
{
|
||||
Name: "kubernetesAuthzRolesApi",
|
||||
Description: "Registers AuthZ Roles /apis endpoint",
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: identityAccessTeam,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
{
|
||||
Name: "kubernetesAuthzRoleBindingsApi",
|
||||
Description: "Registers AuthZ Role Bindings /apis endpoint",
|
||||
Stage: FeatureStageExperimental,
|
||||
Owner: identityAccessTeam,
|
||||
HideFromDocs: true,
|
||||
},
|
||||
{
|
||||
Name: "kubernetesAuthnMutation",
|
||||
Description: "Enables create, delete, and update mutations for resources owned by IAM identity",
|
||||
@@ -1859,14 +1873,6 @@ var (
|
||||
Expression: "false",
|
||||
RequiresRestart: true,
|
||||
},
|
||||
{
|
||||
Name: "tempoSearchBackendMigration",
|
||||
Description: "Run search queries through the tempo backend",
|
||||
Stage: FeatureStageGeneralAvailability,
|
||||
Owner: grafanaOSSBigTent,
|
||||
Expression: "false",
|
||||
RequiresRestart: true,
|
||||
},
|
||||
{
|
||||
Name: "cdnPluginsLoadFirst",
|
||||
Description: "Prioritize loading plugins from the CDN before other sources",
|
||||
|
||||
Generated
+4
-3
@@ -150,7 +150,6 @@ alertingQueryAndExpressionsStepMode,GA,@grafana/alerting-squad,false,false,true
|
||||
improvedExternalSessionHandling,GA,@grafana/identity-access-team,false,false,false
|
||||
useSessionStorageForRedirection,GA,@grafana/identity-access-team,false,false,false
|
||||
rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false
|
||||
unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false
|
||||
unifiedStorageSearchSprinkles,experimental,@grafana/search-and-storage,false,false,false
|
||||
managedDualWriter,experimental,@grafana/search-and-storage,false,false,false
|
||||
pluginsSriChecks,GA,@grafana/plugins-platform-backend,false,false,false
|
||||
@@ -217,10 +216,13 @@ pluginsAutoUpdate,experimental,@grafana/plugins-platform-backend,false,false,fal
|
||||
alertingListViewV2PreviewToggle,privatePreview,@grafana/alerting-squad,false,false,true
|
||||
alertRuleUseFiredAtForStartsAt,experimental,@grafana/alerting-squad,false,false,false
|
||||
alertingBulkActionsInUI,GA,@grafana/alerting-squad,false,false,true
|
||||
kubernetesAuthzApis,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthzApis,deprecated,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthZHandlerRedirect,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthzResourcePermissionApis,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthzZanzanaSync,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthzCoreRolesApi,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthzRolesApi,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthzRoleBindingsApi,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesAuthnMutation,experimental,@grafana/identity-access-team,false,false,false
|
||||
kubernetesExternalGroupMapping,experimental,@grafana/identity-access-team,false,false,false
|
||||
restoreDashboards,experimental,@grafana/grafana-search-navigate-organise,false,false,false
|
||||
@@ -253,7 +255,6 @@ graphiteBackendMode,privatePreview,@grafana/partner-datasources,false,false,fals
|
||||
azureResourcePickerUpdates,GA,@grafana/partner-datasources,false,false,true
|
||||
prometheusTypeMigration,experimental,@grafana/partner-datasources,false,true,false
|
||||
pluginContainers,privatePreview,@grafana/plugins-platform-backend,false,true,false
|
||||
tempoSearchBackendMigration,GA,@grafana/oss-big-tent,false,true,false
|
||||
cdnPluginsLoadFirst,experimental,@grafana/plugins-platform-backend,false,false,false
|
||||
cdnPluginsUrls,experimental,@grafana/plugins-platform-backend,false,false,false
|
||||
pluginInstallAPISync,experimental,@grafana/plugins-platform-backend,false,false,false
|
||||
|
||||
|
Generated
+13
-9
@@ -455,10 +455,6 @@ const (
|
||||
// Enables the new role picker drawer design
|
||||
FlagRolePickerDrawer = "rolePickerDrawer"
|
||||
|
||||
// FlagUnifiedStorageSearch
|
||||
// Enable unified storage search
|
||||
FlagUnifiedStorageSearch = "unifiedStorageSearch"
|
||||
|
||||
// FlagUnifiedStorageSearchSprinkles
|
||||
// Enable sprinkles on unified storage search
|
||||
FlagUnifiedStorageSearchSprinkles = "unifiedStorageSearchSprinkles"
|
||||
@@ -631,7 +627,7 @@ const (
|
||||
FlagAlertRuleUseFiredAtForStartsAt = "alertRuleUseFiredAtForStartsAt"
|
||||
|
||||
// FlagKubernetesAuthzApis
|
||||
// Registers AuthZ /apis endpoint
|
||||
// Deprecated: Use kubernetesAuthzCoreRolesApi, kubernetesAuthzRolesApi, and kubernetesAuthzRoleBindingsApi instead
|
||||
FlagKubernetesAuthzApis = "kubernetesAuthzApis"
|
||||
|
||||
// FlagKubernetesAuthZHandlerRedirect
|
||||
@@ -646,6 +642,18 @@ const (
|
||||
// Enable sync of Zanzana authorization store on AuthZ CRD mutations
|
||||
FlagKubernetesAuthzZanzanaSync = "kubernetesAuthzZanzanaSync"
|
||||
|
||||
// FlagKubernetesAuthzCoreRolesApi
|
||||
// Registers AuthZ Core Roles /apis endpoint
|
||||
FlagKubernetesAuthzCoreRolesApi = "kubernetesAuthzCoreRolesApi"
|
||||
|
||||
// FlagKubernetesAuthzRolesApi
|
||||
// Registers AuthZ Roles /apis endpoint
|
||||
FlagKubernetesAuthzRolesApi = "kubernetesAuthzRolesApi"
|
||||
|
||||
// FlagKubernetesAuthzRoleBindingsApi
|
||||
// Registers AuthZ Role Bindings /apis endpoint
|
||||
FlagKubernetesAuthzRoleBindingsApi = "kubernetesAuthzRoleBindingsApi"
|
||||
|
||||
// FlagKubernetesAuthnMutation
|
||||
// Enables create, delete, and update mutations for resources owned by IAM identity
|
||||
FlagKubernetesAuthnMutation = "kubernetesAuthnMutation"
|
||||
@@ -730,10 +738,6 @@ const (
|
||||
// Enables running plugins in containers
|
||||
FlagPluginContainers = "pluginContainers"
|
||||
|
||||
// FlagTempoSearchBackendMigration
|
||||
// Run search queries through the tempo backend
|
||||
FlagTempoSearchBackendMigration = "tempoSearchBackendMigration"
|
||||
|
||||
// FlagCdnPluginsLoadFirst
|
||||
// Prioritize loading plugins from the CDN before other sources
|
||||
FlagCdnPluginsLoadFirst = "cdnPluginsLoadFirst"
|
||||
|
||||
+47
-4
@@ -1951,11 +1951,27 @@
|
||||
{
|
||||
"metadata": {
|
||||
"name": "kubernetesAuthzApis",
|
||||
"resourceVersion": "1764664939750",
|
||||
"creationTimestamp": "2025-06-18T07:43:01Z"
|
||||
"resourceVersion": "1767954559317",
|
||||
"creationTimestamp": "2025-06-18T07:43:01Z",
|
||||
"annotations": {
|
||||
"grafana.app/updatedTimestamp": "2026-01-09 10:29:19.317164 +0000 UTC"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"description": "Registers AuthZ /apis endpoint",
|
||||
"description": "Deprecated: Use kubernetesAuthzCoreRolesApi, kubernetesAuthzRolesApi, and kubernetesAuthzRoleBindingsApi instead",
|
||||
"stage": "deprecated",
|
||||
"codeowner": "@grafana/identity-access-team",
|
||||
"hideFromDocs": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "kubernetesAuthzCoreRolesApi",
|
||||
"resourceVersion": "1767954459090",
|
||||
"creationTimestamp": "2026-01-09T10:27:39Z"
|
||||
},
|
||||
"spec": {
|
||||
"description": "Registers AuthZ Core Roles /apis endpoint",
|
||||
"stage": "experimental",
|
||||
"codeowner": "@grafana/identity-access-team",
|
||||
"hideFromDocs": true
|
||||
@@ -1975,6 +1991,32 @@
|
||||
"hideFromDocs": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "kubernetesAuthzRoleBindingsApi",
|
||||
"resourceVersion": "1767954459090",
|
||||
"creationTimestamp": "2026-01-09T10:27:39Z"
|
||||
},
|
||||
"spec": {
|
||||
"description": "Registers AuthZ Role Bindings /apis endpoint",
|
||||
"stage": "experimental",
|
||||
"codeowner": "@grafana/identity-access-team",
|
||||
"hideFromDocs": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "kubernetesAuthzRolesApi",
|
||||
"resourceVersion": "1767954459090",
|
||||
"creationTimestamp": "2026-01-09T10:27:39Z"
|
||||
},
|
||||
"spec": {
|
||||
"description": "Registers AuthZ Roles /apis endpoint",
|
||||
"stage": "experimental",
|
||||
"codeowner": "@grafana/identity-access-team",
|
||||
"hideFromDocs": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "kubernetesAuthzZanzanaSync",
|
||||
@@ -3655,7 +3697,8 @@
|
||||
"metadata": {
|
||||
"name": "unifiedStorageSearch",
|
||||
"resourceVersion": "1764664939750",
|
||||
"creationTimestamp": "2024-09-30T19:46:14Z"
|
||||
"creationTimestamp": "2024-09-30T19:46:14Z",
|
||||
"deletionTimestamp": "2026-01-12T10:02:12Z"
|
||||
},
|
||||
"spec": {
|
||||
"description": "Enable unified storage search",
|
||||
|
||||
@@ -16,13 +16,6 @@ import (
|
||||
"github.com/grafana/alerting/receivers/schema"
|
||||
)
|
||||
|
||||
// GetReceiverQuery represents a query for a single receiver.
|
||||
type GetReceiverQuery struct {
|
||||
OrgID int64
|
||||
Name string
|
||||
Decrypt bool
|
||||
}
|
||||
|
||||
// GetReceiversQuery represents a query for receiver groups.
|
||||
type GetReceiversQuery struct {
|
||||
OrgID int64
|
||||
|
||||
@@ -378,3 +378,29 @@ func EncryptedReceivers(receivers []*definitions.PostableApiReceiver, encryptFn
|
||||
}
|
||||
return encrypted, nil
|
||||
}
|
||||
|
||||
// DecryptIntegrationSettings returns a function to decrypt integration settings.
|
||||
func DecryptIntegrationSettings(ctx context.Context, ss secretService) models.DecryptFn {
|
||||
return func(value string) (string, error) {
|
||||
decoded, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
decrypted, err := ss.Decrypt(ctx, decoded)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(decrypted), nil
|
||||
}
|
||||
}
|
||||
|
||||
// EncryptIntegrationSettings returns a function to encrypt integration settings.
|
||||
func EncryptIntegrationSettings(ctx context.Context, ss secretService) models.EncryptFn {
|
||||
return func(payload string) (string, error) {
|
||||
encrypted, err := ss.Encrypt(ctx, []byte(payload), secrets.WithoutScope())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encrypted), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,37 @@
|
||||
package notifier
|
||||
|
||||
import "github.com/grafana/grafana/pkg/apimachinery/errutil"
|
||||
import (
|
||||
"errors"
|
||||
"slices"
|
||||
|
||||
"github.com/grafana/alerting/receivers/schema"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/errutil"
|
||||
)
|
||||
|
||||
// WithPublicError sets the public message of an errutil error to the error message.
|
||||
func WithPublicError(err errutil.Error) error {
|
||||
err.PublicMessage = err.Error()
|
||||
return err
|
||||
}
|
||||
|
||||
// If provided error is errutil.Error it appends fields that caused the error to public payload
|
||||
func makeProtectedFieldsAuthzError(err error, diff map[string][]schema.IntegrationFieldPath) error {
|
||||
var authzErr errutil.Error
|
||||
if !errors.As(err, &authzErr) {
|
||||
return err
|
||||
}
|
||||
if authzErr.PublicPayload == nil {
|
||||
authzErr.PublicPayload = map[string]interface{}{}
|
||||
}
|
||||
fields := make(map[string][]string, len(diff))
|
||||
for field, paths := range diff {
|
||||
fields[field] = make([]string, len(paths))
|
||||
for i, path := range paths {
|
||||
fields[field][i] = path.String()
|
||||
}
|
||||
slices.Sort(fields[field])
|
||||
}
|
||||
authzErr.PublicPayload["changed_protected_fields"] = fields
|
||||
return authzErr
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package notifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -133,30 +132,33 @@ func (rs *ReceiverService) loadProvenances(ctx context.Context, orgID int64) (ma
|
||||
return rs.provisioningStore.GetProvenances(ctx, orgID, (&models.Integration{}).ResourceType())
|
||||
}
|
||||
|
||||
// GetReceiver returns a receiver by name.
|
||||
// GetReceiver returns a receiver by its UID.
|
||||
// The receiver's secure settings are decrypted if requested and the user has access to do so.
|
||||
func (rs *ReceiverService) GetReceiver(ctx context.Context, q models.GetReceiverQuery, user identity.Requester) (*models.Receiver, error) {
|
||||
func (rs *ReceiverService) GetReceiver(ctx context.Context, uid string, decrypt bool, user identity.Requester) (*models.Receiver, error) {
|
||||
if user == nil {
|
||||
return nil, errors.New("user is required")
|
||||
}
|
||||
ctx, span := rs.tracer.Start(ctx, "alerting.receivers.get", trace.WithAttributes(
|
||||
attribute.Int64("query_org_id", q.OrgID),
|
||||
attribute.String("query_name", q.Name),
|
||||
attribute.Bool("query_decrypt", q.Decrypt),
|
||||
attribute.Int64("query_org_id", user.GetOrgID()),
|
||||
attribute.String("query_uid", uid),
|
||||
attribute.Bool("query_decrypt", decrypt),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
revision, err := rs.cfgStore.Get(ctx, q.OrgID)
|
||||
revision, err := rs.cfgStore.Get(ctx, user.GetOrgID())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prov, err := rs.loadProvenances(ctx, q.OrgID)
|
||||
prov, err := rs.loadProvenances(ctx, user.GetOrgID())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rcv, err := revision.GetReceiver(legacy_storage.NameToUid(q.Name), prov)
|
||||
rcv, err := revision.GetReceiver(uid, prov)
|
||||
if err != nil {
|
||||
if errors.Is(err, legacy_storage.ErrReceiverNotFound) && rs.includeImported {
|
||||
imported := rs.getImportedReceivers(ctx, span, []string{legacy_storage.NameToUid(q.Name)}, revision)
|
||||
imported := rs.getImportedReceivers(ctx, span, []string{uid}, revision)
|
||||
if len(imported) > 0 {
|
||||
rcv = imported[0]
|
||||
}
|
||||
@@ -171,14 +173,14 @@ func (rs *ReceiverService) GetReceiver(ctx context.Context, q models.GetReceiver
|
||||
))
|
||||
|
||||
auth := rs.authz.AuthorizeReadDecrypted
|
||||
if !q.Decrypt {
|
||||
if !decrypt {
|
||||
auth = rs.authz.AuthorizeRead
|
||||
}
|
||||
if err := auth(ctx, user, rcv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if q.Decrypt {
|
||||
if decrypt {
|
||||
err := rcv.Decrypt(rs.decryptor(ctx))
|
||||
if err != nil {
|
||||
rs.log.FromContext(ctx).Warn("Failed to decrypt secure settings", "name", rcv.Name, "error", err)
|
||||
@@ -684,28 +686,12 @@ func (rs *ReceiverService) deleteProvenances(ctx context.Context, orgID int64, i
|
||||
|
||||
// decryptor returns a models.DecryptFn that decrypts a secure setting. If decryption fails, the fallback value is used.
|
||||
func (rs *ReceiverService) decryptor(ctx context.Context) models.DecryptFn {
|
||||
return func(value string) (string, error) {
|
||||
decoded, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
decrypted, err := rs.encryptionService.Decrypt(ctx, decoded)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(decrypted), nil
|
||||
}
|
||||
return DecryptIntegrationSettings(ctx, rs.encryptionService)
|
||||
}
|
||||
|
||||
// encryptor creates an encrypt function that delegates to secrets.Service and returns the base64 encoded result.
|
||||
func (rs *ReceiverService) encryptor(ctx context.Context) models.EncryptFn {
|
||||
return func(payload string) (string, error) {
|
||||
s, err := rs.encryptionService.Encrypt(ctx, []byte(payload), secrets.WithoutScope())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(s), nil
|
||||
}
|
||||
return EncryptIntegrationSettings(ctx, rs.encryptionService)
|
||||
}
|
||||
|
||||
// checkOptimisticConcurrency checks if the existing receiver's version matches the desired version.
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
package notifier
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"slices"
|
||||
|
||||
"github.com/grafana/alerting/receivers/schema"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/errutil"
|
||||
)
|
||||
|
||||
func makeProtectedFieldsAuthzError(err error, diff map[string][]schema.IntegrationFieldPath) error {
|
||||
var authzErr errutil.Error
|
||||
if !errors.As(err, &authzErr) {
|
||||
return err
|
||||
}
|
||||
if authzErr.PublicPayload == nil {
|
||||
authzErr.PublicPayload = map[string]interface{}{}
|
||||
}
|
||||
fields := make(map[string][]string, len(diff))
|
||||
for field, paths := range diff {
|
||||
fields[field] = make([]string, len(paths))
|
||||
for i, path := range paths {
|
||||
fields[field][i] = path.String()
|
||||
}
|
||||
slices.Sort(fields[field])
|
||||
}
|
||||
authzErr.PublicPayload["changed_protected_fields"] = fields
|
||||
return authzErr
|
||||
}
|
||||
@@ -50,7 +50,7 @@ func TestIntegrationReceiverService_GetReceiver(t *testing.T) {
|
||||
|
||||
t.Run("service gets receiver from AM config", func(t *testing.T) {
|
||||
sut := createReceiverServiceSut(t, secretsService)
|
||||
recv, err := sut.GetReceiver(context.Background(), singleQ(1, "slack receiver"), redactedUser)
|
||||
recv, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid("slack receiver"), false, redactedUser)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "slack receiver", recv.Name)
|
||||
require.Len(t, recv.Integrations, 1)
|
||||
@@ -60,7 +60,7 @@ func TestIntegrationReceiverService_GetReceiver(t *testing.T) {
|
||||
t.Run("service returns error when receiver does not exist", func(t *testing.T) {
|
||||
sut := createReceiverServiceSut(t, secretsService)
|
||||
|
||||
_, err := sut.GetReceiver(context.Background(), singleQ(1, "receiver1"), redactedUser)
|
||||
_, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid("receiver1"), false, redactedUser)
|
||||
require.ErrorIs(t, err, legacy_storage.ErrReceiverNotFound)
|
||||
})
|
||||
|
||||
@@ -68,7 +68,7 @@ func TestIntegrationReceiverService_GetReceiver(t *testing.T) {
|
||||
t.Run("gets imported receivers", func(t *testing.T) {
|
||||
sut := createReceiverServiceSut(t, secretsService, withImportedIncluded)
|
||||
|
||||
recv, err := sut.GetReceiver(context.Background(), singleQ(1, "receiver1"), redactedUser)
|
||||
recv, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid("receiver1"), false, redactedUser)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, models.ResourceOriginImported, recv.Origin)
|
||||
assert.Equal(t, "receiver1", recv.Name)
|
||||
@@ -81,9 +81,9 @@ func TestIntegrationReceiverService_GetReceiver(t *testing.T) {
|
||||
|
||||
t.Run("falls to only Grafana if cannot read imported receivers", func(t *testing.T) {
|
||||
sut := createReceiverServiceSut(t, secretsService, withImportedIncluded, withInvalidExtraConfig)
|
||||
_, err := sut.GetReceiver(context.Background(), singleQ(1, "receiver1"), redactedUser)
|
||||
_, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid("receiver1"), false, redactedUser)
|
||||
require.ErrorIs(t, err, legacy_storage.ErrReceiverNotFound)
|
||||
_, err = sut.GetReceiver(context.Background(), singleQ(1, "slack receiver"), redactedUser)
|
||||
_, err = sut.GetReceiver(context.Background(), legacy_storage.NameToUid("slack receiver"), false, redactedUser)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
})
|
||||
@@ -187,12 +187,6 @@ func TestIntegrationReceiverService_DecryptRedact(t *testing.T) {
|
||||
user: readUser,
|
||||
err: "[alerting.unauthorized] user is not authorized to read any decrypted receiver",
|
||||
},
|
||||
{
|
||||
name: "service returns error if user is nil and decrypt is true",
|
||||
decrypt: true,
|
||||
user: nil,
|
||||
err: "[alerting.unauthorized] user is not authorized to read any decrypted receiver",
|
||||
},
|
||||
{
|
||||
name: "service decrypts receivers with permission",
|
||||
decrypt: true,
|
||||
@@ -224,18 +218,16 @@ func TestIntegrationReceiverService_DecryptRedact(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
for _, o := range origin {
|
||||
for _, method := range getMethods {
|
||||
t.Run(fmt.Sprintf("%s %s", tc.name, method), func(t *testing.T) {
|
||||
for _, method := range getMethods {
|
||||
t.Run(fmt.Sprintf("%s %s", tc.name, method), func(t *testing.T) {
|
||||
for _, o := range origin {
|
||||
t.Run(fmt.Sprintf("%s %s (%s)", tc.name, method, o.origin), func(t *testing.T) {
|
||||
sut := createReceiverServiceSut(t, secretsService, o.opts...)
|
||||
|
||||
var res *models.Receiver
|
||||
var err error
|
||||
if method == "single" {
|
||||
q := singleQ(1, o.receiver)
|
||||
q.Decrypt = tc.decrypt
|
||||
res, err = sut.GetReceiver(context.Background(), q, tc.user)
|
||||
res, err = sut.GetReceiver(context.Background(), legacy_storage.NameToUid(o.receiver), tc.decrypt, tc.user)
|
||||
} else {
|
||||
q := multiQ(1, o.receiver)
|
||||
q.Decrypt = tc.decrypt
|
||||
@@ -267,8 +259,8 @@ func TestIntegrationReceiverService_DecryptRedact(t *testing.T) {
|
||||
require.NotEqual(t, o.decryptedSettingValue, res.Integrations[0].SecureSettings[o.secureSettingKey])
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -412,8 +404,7 @@ func TestReceiverService_Delete(t *testing.T) {
|
||||
// Ensure receiver saved to store is correct.
|
||||
name, err := legacy_storage.UidToName(tc.deleteUID)
|
||||
require.NoError(t, err)
|
||||
q := models.GetReceiverQuery{OrgID: tc.user.GetOrgID(), Name: name}
|
||||
_, err = sut.GetReceiver(context.Background(), q, writer)
|
||||
_, err = sut.GetReceiver(context.Background(), legacy_storage.NameToUid(name), false, writer)
|
||||
assert.ErrorIs(t, err, legacy_storage.ErrReceiverNotFound)
|
||||
|
||||
provenances, err := sut.provisioningStore.GetProvenances(context.Background(), tc.user.GetOrgID(), (&definitions.EmbeddedContactPoint{}).ResourceType())
|
||||
@@ -626,8 +617,7 @@ func TestReceiverService_Create(t *testing.T) {
|
||||
assert.Equal(t, tc.expectedCreate, *created)
|
||||
|
||||
// Ensure receiver saved to store is correct.
|
||||
q := models.GetReceiverQuery{OrgID: tc.user.GetOrgID(), Name: tc.receiver.Name, Decrypt: true}
|
||||
stored, err := sut.GetReceiver(context.Background(), q, decryptUser)
|
||||
stored, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid(tc.receiver.Name), true, decryptUser)
|
||||
require.NoError(t, err)
|
||||
decrypted := models.CopyReceiverWith(tc.expectedCreate, models.ReceiverMuts.Decrypted(models.Base64Decrypt))
|
||||
decrypted.Version = tc.expectedCreate.Version // Version is calculated before decryption.
|
||||
@@ -931,8 +921,7 @@ func TestReceiverService_Update(t *testing.T) {
|
||||
assert.Equal(t, tc.expectedUpdate, *updated)
|
||||
|
||||
// Ensure receiver saved to store is correct.
|
||||
q := models.GetReceiverQuery{OrgID: tc.user.GetOrgID(), Name: tc.receiver.Name, Decrypt: true}
|
||||
stored, err := sut.GetReceiver(context.Background(), q, decryptUser)
|
||||
stored, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid(tc.receiver.Name), true, decryptUser)
|
||||
require.NoError(t, err)
|
||||
decrypted := models.CopyReceiverWith(tc.expectedUpdate, models.ReceiverMuts.Decrypted(models.Base64Decrypt))
|
||||
decrypted.Version = tc.expectedUpdate.Version // Version is calculated before decryption.
|
||||
@@ -1054,7 +1043,7 @@ func TestReceiverService_UpdateReceiverName(t *testing.T) {
|
||||
sut.ruleNotificationsStore = ruleStore
|
||||
|
||||
newReceiverName = "receiver1"
|
||||
actual, err := sut.GetReceiver(context.Background(), models.GetReceiverQuery{OrgID: writer.GetOrgID(), Name: newReceiverName}, writer)
|
||||
actual, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid(newReceiverName), false, writer)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, models.ResourceOriginImported, actual.Origin)
|
||||
require.Equal(t, newReceiverName, actual.Name)
|
||||
@@ -1067,7 +1056,7 @@ func TestReceiverService_UpdateReceiverName(t *testing.T) {
|
||||
require.NotEqual(t, actual, recv)
|
||||
require.Equal(t, models.ResourceOriginGrafana, recv.Origin)
|
||||
|
||||
actual, err = sut.GetReceiver(context.Background(), models.GetReceiverQuery{OrgID: writer.GetOrgID(), Name: newReceiverName}, writer)
|
||||
actual, err = sut.GetReceiver(context.Background(), legacy_storage.NameToUid(newReceiverName), false, writer)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, recv.Name, actual.Name)
|
||||
})
|
||||
@@ -1185,7 +1174,7 @@ func TestReceiverServiceAC_Read(t *testing.T) {
|
||||
return false
|
||||
}
|
||||
for _, recv := range allReceivers() {
|
||||
response, err := sut.GetReceiver(context.Background(), singleQ(orgId, recv.Name), usr)
|
||||
response, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid(recv.Name), false, usr)
|
||||
if isVisible(recv.UID) {
|
||||
require.NoErrorf(t, err, "receiver '%s' should be visible, but isn't", recv.Name)
|
||||
assert.NotNil(t, response)
|
||||
@@ -1207,7 +1196,7 @@ func TestReceiverServiceAC_Read(t *testing.T) {
|
||||
}
|
||||
sut.authz = ac.NewReceiverAccess[*models.Receiver](acimpl.ProvideAccessControl(featuremgmt.WithFeatures()), true)
|
||||
for _, recv := range allReceivers() {
|
||||
response, err := sut.GetReceiver(context.Background(), singleQ(orgId, recv.Name), usr)
|
||||
response, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid(recv.Name), false, usr)
|
||||
if isVisibleInProvisioning(recv.UID) {
|
||||
require.NoErrorf(t, err, "receiver '%s' should be visible, but isn't", recv.Name)
|
||||
assert.NotNil(t, response)
|
||||
@@ -1766,7 +1755,7 @@ func TestReceiverService_AccessControlMetadata(t *testing.T) {
|
||||
},
|
||||
}}
|
||||
|
||||
r, err := sut.GetReceiver(context.Background(), models.GetReceiverQuery{OrgID: 1, Name: "receiver1"}, admin)
|
||||
r, err := sut.GetReceiver(context.Background(), legacy_storage.NameToUid("receiver1"), false, admin)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("should override metadata for imported receivers", func(t *testing.T) {
|
||||
@@ -1842,13 +1831,6 @@ func createEncryptedConfig(t *testing.T, secretService secretService, extraConfi
|
||||
return string(bytes)
|
||||
}
|
||||
|
||||
func singleQ(orgID int64, name string) models.GetReceiverQuery {
|
||||
return models.GetReceiverQuery{
|
||||
OrgID: orgID,
|
||||
Name: name,
|
||||
}
|
||||
}
|
||||
|
||||
func multiQ(orgID int64, names ...string) models.GetReceiversQuery {
|
||||
return models.GetReceiversQuery{
|
||||
OrgID: orgID,
|
||||
|
||||
@@ -40,6 +40,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
||||
remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/sender"
|
||||
"github.com/grafana/grafana/pkg/services/secrets"
|
||||
"github.com/grafana/grafana/pkg/util/cmputil"
|
||||
)
|
||||
|
||||
@@ -57,6 +58,7 @@ func NoopAutogenFn(_ context.Context, _ log.Logger, _ int64, _ *apimodels.Postab
|
||||
}
|
||||
|
||||
type Crypto interface {
|
||||
Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error)
|
||||
Decrypt(ctx context.Context, payload []byte) ([]byte, error)
|
||||
DecryptExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error
|
||||
}
|
||||
@@ -289,20 +291,6 @@ func (am *Alertmanager) isDefaultConfiguration(configHash string) bool {
|
||||
return configHash == am.defaultConfigHash
|
||||
}
|
||||
|
||||
func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn {
|
||||
return func(value string) (string, error) {
|
||||
decoded, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
decrypted, err := crypto.Decrypt(ctx, decoded)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(decrypted), nil
|
||||
}
|
||||
}
|
||||
|
||||
// buildConfiguration takes a raw Alertmanager configuration and returns a config that the remote Alertmanager can use.
|
||||
// It parses the initial configuration, adds auto-generated routes, decrypts receivers, and merges the extra configs.
|
||||
func (am *Alertmanager) buildConfiguration(ctx context.Context, raw []byte, createdAtEpoch int64, autogenInvalidReceiverAction notifier.InvalidReceiversAction) (remoteClient.UserGrafanaConfig, error) {
|
||||
@@ -317,7 +305,7 @@ func (am *Alertmanager) buildConfiguration(ctx context.Context, raw []byte, crea
|
||||
}
|
||||
|
||||
// Decrypt the receivers in the configuration.
|
||||
decryptedReceivers, err := notifier.DecryptedReceivers(c.AlertmanagerConfig.Receivers, decrypter(ctx, am.crypto))
|
||||
decryptedReceivers, err := notifier.DecryptedReceivers(c.AlertmanagerConfig.Receivers, notifier.DecryptIntegrationSettings(ctx, am.crypto))
|
||||
if err != nil {
|
||||
return remoteClient.UserGrafanaConfig{}, fmt.Errorf("unable to decrypt receivers: %w", err)
|
||||
}
|
||||
@@ -619,7 +607,7 @@ func (am *Alertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver,
|
||||
}
|
||||
|
||||
func (am *Alertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error) {
|
||||
decryptedReceivers, err := notifier.DecryptedReceivers(c.Receivers, decrypter(ctx, am.crypto))
|
||||
decryptedReceivers, err := notifier.DecryptedReceivers(c.Receivers, notifier.DecryptIntegrationSettings(ctx, am.crypto))
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to decrypt receivers: %w", err)
|
||||
}
|
||||
|
||||
@@ -43,7 +43,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
|
||||
ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
|
||||
"github.com/grafana/grafana/pkg/services/secrets"
|
||||
"github.com/grafana/grafana/pkg/services/secrets/database"
|
||||
"github.com/grafana/grafana/pkg/services/secrets/fakes"
|
||||
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
|
||||
@@ -298,13 +297,7 @@ func TestIntegrationApplyConfig(t *testing.T) {
|
||||
var c apimodels.PostableUserConfig
|
||||
require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &c))
|
||||
secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(c.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
|
||||
encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encrypted), nil
|
||||
})
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(c.AlertmanagerConfig.Receivers, notifier.EncryptIntegrationSettings(context.Background(), secretsService))
|
||||
c.AlertmanagerConfig.Receivers = encryptedReceivers
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -462,13 +455,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
|
||||
// Create a config with correctly encrypted and encoded secrets.
|
||||
var inputCfg apimodels.PostableUserConfig
|
||||
require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &inputCfg))
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(inputCfg.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
|
||||
encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encrypted), nil
|
||||
})
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(inputCfg.AlertmanagerConfig.Receivers, notifier.EncryptIntegrationSettings(context.Background(), secretsService))
|
||||
inputCfg.AlertmanagerConfig.Receivers = encryptedReceivers
|
||||
require.NoError(t, err)
|
||||
testGrafanaConfigWithEncryptedSecret, err := json.Marshal(inputCfg)
|
||||
@@ -663,13 +650,7 @@ func Test_TestReceiversDecryptsSecureSettings(t *testing.T) {
|
||||
|
||||
var inputCfg apimodels.PostableUserConfig
|
||||
require.NoError(t, json.Unmarshal([]byte(testGrafanaConfigWithSecret), &inputCfg))
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(inputCfg.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
|
||||
encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encrypted), nil
|
||||
})
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(inputCfg.AlertmanagerConfig.Receivers, notifier.EncryptIntegrationSettings(context.Background(), secretsService))
|
||||
inputCfg.AlertmanagerConfig.Receivers = encryptedReceivers
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1037,13 +1018,7 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
|
||||
{
|
||||
postableCfg, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
|
||||
require.NoError(t, err)
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(postableCfg.AlertmanagerConfig.Receivers, func(payload string) (string, error) {
|
||||
encrypted, err := secretsService.Encrypt(context.Background(), []byte(payload), secrets.WithoutScope())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encrypted), nil
|
||||
})
|
||||
encryptedReceivers, err := notifier.EncryptedReceivers(postableCfg.AlertmanagerConfig.Receivers, notifier.EncryptIntegrationSettings(context.Background(), secretsService))
|
||||
postableCfg.AlertmanagerConfig.Receivers = encryptedReceivers
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ type OrgUserDTO struct {
|
||||
Role string `json:"role"`
|
||||
LastSeenAt time.Time `json:"lastSeenAt"`
|
||||
Updated time.Time `json:"-"`
|
||||
Created time.Time `json:"-"`
|
||||
Created time.Time `json:"created"`
|
||||
LastSeenAtAge string `json:"lastSeenAtAge"`
|
||||
AccessControl map[string]bool `json:"accessControl,omitempty"`
|
||||
IsDisabled bool `json:"isDisabled"`
|
||||
|
||||
@@ -100,6 +100,9 @@ func (d *DsLookup) ByRef(ref *DataSourceRef) *DataSourceRef {
|
||||
if ref == nil {
|
||||
return d.defaultDS
|
||||
}
|
||||
if ref.UID == "default" && ref.Type == "" {
|
||||
return d.defaultDS
|
||||
}
|
||||
|
||||
key := ""
|
||||
if ref.UID != "" {
|
||||
@@ -117,7 +120,13 @@ func (d *DsLookup) ByRef(ref *DataSourceRef) *DataSourceRef {
|
||||
return ds
|
||||
}
|
||||
|
||||
return d.byName[key]
|
||||
ds, ok = d.byName[key]
|
||||
if ok {
|
||||
return ds
|
||||
}
|
||||
|
||||
// With nothing was found (or configured), use the original reference
|
||||
return ref
|
||||
}
|
||||
|
||||
func (d *DsLookup) ByType(dsType string) []DataSourceRef {
|
||||
|
||||
+4
-4
@@ -4,8 +4,8 @@
|
||||
"tags": null,
|
||||
"datasource": [
|
||||
{
|
||||
"uid": "default.uid",
|
||||
"type": "default.type"
|
||||
"uid": "000000001",
|
||||
"type": "graphite"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
@@ -16,8 +16,8 @@
|
||||
"libraryPanel": "dfkljg98345dkf",
|
||||
"datasource": [
|
||||
{
|
||||
"uid": "default.uid",
|
||||
"type": "default.type"
|
||||
"uid": "000000001",
|
||||
"type": "graphite"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package dashboard
|
||||
|
||||
import "iter"
|
||||
|
||||
type PanelSummaryInfo struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
@@ -30,3 +32,20 @@ type DashboardSummaryInfo struct {
|
||||
Refresh string `json:"refresh,omitempty"`
|
||||
ReadOnly bool `json:"readOnly,omitempty"` // editable = false
|
||||
}
|
||||
|
||||
func (d *DashboardSummaryInfo) PanelIterator() iter.Seq[PanelSummaryInfo] {
|
||||
return func(yield func(PanelSummaryInfo) bool) {
|
||||
for _, p := range d.Panels {
|
||||
if len(p.Collapsed) > 0 {
|
||||
for _, c := range p.Collapsed {
|
||||
if !yield(c) { // NOTE, rows can only be one level deep!
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if !yield(p) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,6 +146,7 @@ type UserSearchHitDTO struct {
|
||||
LastSeenAtAge string `json:"lastSeenAtAge"`
|
||||
AuthLabels []string `json:"authLabels"`
|
||||
AuthModule AuthModuleConversion `json:"-"`
|
||||
Created time.Time `json:"created" xorm:"created"`
|
||||
}
|
||||
|
||||
type GetUserProfileQuery struct {
|
||||
|
||||
@@ -541,7 +541,7 @@ func (ss *sqlStore) Search(ctx context.Context, query *user.SearchUsersQuery) (*
|
||||
sess.Limit(query.Limit, offset)
|
||||
}
|
||||
|
||||
sess.Cols("u.id", "u.uid", "u.email", "u.name", "u.login", "u.is_admin", "u.is_disabled", "u.last_seen_at", "user_auth.auth_module", "u.is_provisioned")
|
||||
sess.Cols("u.id", "u.uid", "u.email", "u.name", "u.login", "u.is_admin", "u.is_disabled", "u.last_seen_at", "user_auth.auth_module", "u.is_provisioned", "u.created")
|
||||
|
||||
if len(query.SortOpts) > 0 {
|
||||
for i := range query.SortOpts {
|
||||
|
||||
@@ -236,7 +236,6 @@ kubernetesDashboards = true
|
||||
kubernetesFolders = true
|
||||
unifiedStorage = true
|
||||
unifiedStorageHistoryPruner = true
|
||||
unifiedStorageSearch = true
|
||||
unifiedStorageSearchPermissionFiltering = false
|
||||
unifiedStorageSearchSprinkles = false
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ var _ generic.RESTOptionsGetter = (*RESTOptionsGetter)(nil)
|
||||
type StorageOptionsRegister func(gr schema.GroupResource, opts StorageOptions)
|
||||
|
||||
type RESTOptionsGetter struct {
|
||||
client resource.ResourceClient
|
||||
client resource.StorageClient
|
||||
secrets secret.InlineSecureValueSupport
|
||||
original storagebackend.Config
|
||||
configProvider RestConfigProvider
|
||||
@@ -36,7 +36,7 @@ type RESTOptionsGetter struct {
|
||||
}
|
||||
|
||||
func NewRESTOptionsGetterForClient(
|
||||
client resource.ResourceClient,
|
||||
client resource.StorageClient,
|
||||
secrets secret.InlineSecureValueSupport,
|
||||
original storagebackend.Config,
|
||||
configProvider RestConfigProvider,
|
||||
@@ -79,7 +79,7 @@ func NewRESTOptionsGetterMemory(originalStorageConfig storagebackend.Config, sec
|
||||
}
|
||||
|
||||
return NewRESTOptionsGetterForClient(
|
||||
resource.NewLocalResourceClient(server),
|
||||
resource.NewLocalResourceClient(server, nil),
|
||||
secrets,
|
||||
originalStorageConfig,
|
||||
nil,
|
||||
@@ -118,7 +118,7 @@ func NewRESTOptionsGetterForFileXX(path string,
|
||||
}
|
||||
|
||||
return NewRESTOptionsGetterForClient(
|
||||
resource.NewLocalResourceClient(server),
|
||||
resource.NewLocalResourceClient(server, nil),
|
||||
nil, // secrets
|
||||
originalStorageConfig,
|
||||
nil,
|
||||
|
||||
@@ -88,7 +88,7 @@ type Storage struct {
|
||||
trigger storage.IndexerFuncs
|
||||
indexers *cache.Indexers
|
||||
|
||||
store resource.ResourceClient
|
||||
store resource.StorageClient
|
||||
getKey func(string) (*resourcepb.ResourceKey, error)
|
||||
snowflake *snowflake.Node // used to enforce internal ids
|
||||
configProvider RestConfigProvider // used for provisioning
|
||||
@@ -112,7 +112,7 @@ type RestConfigProvider interface {
|
||||
// NewStorage instantiates a new Storage.
|
||||
func NewStorage(
|
||||
config *storagebackend.ConfigForResource,
|
||||
store resource.ResourceClient,
|
||||
store resource.StorageClient,
|
||||
keyFunc func(obj runtime.Object) (string, error),
|
||||
keyParser func(key string) (*resourcepb.ResourceKey, error),
|
||||
newFunc func() runtime.Object,
|
||||
|
||||
@@ -156,7 +156,7 @@ func testSetup(t testing.TB, opts ...setupOption) (context.Context, storage.Inte
|
||||
default:
|
||||
t.Fatalf("unsupported storage type: %s", setupOpts.storageType)
|
||||
}
|
||||
client := resource.NewLocalResourceClient(server)
|
||||
client := resource.NewLocalResourceClient(server, nil)
|
||||
|
||||
config := storagebackend.NewDefaultConfig(setupOpts.prefix, setupOpts.codec)
|
||||
store, destroyFunc, err := apistore.NewStorage(
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/grafana/dskit/grpcclient"
|
||||
"github.com/grafana/dskit/middleware"
|
||||
"github.com/grafana/dskit/services"
|
||||
|
||||
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
|
||||
infraDB "github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
@@ -45,6 +46,7 @@ type Options struct {
|
||||
Authzc types.AccessClient
|
||||
Docs resource.DocumentBuilderSupplier
|
||||
SecureValues secrets.InlineSecureValueSupport
|
||||
Backend resource.StorageBackend // Shared backend to avoid duplicate metrics registration
|
||||
}
|
||||
|
||||
type clientMetrics struct {
|
||||
@@ -66,7 +68,7 @@ func ProvideUnifiedStorageClient(opts *Options,
|
||||
BlobStoreURL: apiserverCfg.Key("blob_url").MustString(""),
|
||||
BlobThresholdBytes: apiserverCfg.Key("blob_threshold_bytes").MustInt(options.BlobThresholdDefault),
|
||||
GrpcClientKeepaliveTime: apiserverCfg.Key("grpc_client_keepalive_time").MustDuration(0),
|
||||
}, opts.Cfg, opts.Features, opts.DB, opts.Tracer, opts.Reg, opts.Authzc, opts.Docs, storageMetrics, indexMetrics, opts.SecureValues)
|
||||
}, opts.Cfg, opts.Features, opts.DB, opts.Tracer, opts.Reg, opts.Authzc, opts.Docs, storageMetrics, indexMetrics, opts.SecureValues, opts.Backend)
|
||||
if err == nil {
|
||||
// Decide whether to disable SQL fallback stats per resource in Mode 5.
|
||||
// Otherwise we would still try to query the legacy SQL database in Mode 5.
|
||||
@@ -102,6 +104,7 @@ func newClient(opts options.StorageOptions,
|
||||
storageMetrics *resource.StorageMetrics,
|
||||
indexMetrics *resource.BleveIndexMetrics,
|
||||
secure secrets.InlineSecureValueSupport,
|
||||
backend resource.StorageBackend,
|
||||
) (resource.ResourceClient, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -135,7 +138,7 @@ func newClient(opts options.StorageOptions,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resource.NewLocalResourceClient(server), nil
|
||||
return resource.NewLocalResourceClient(server, nil), nil
|
||||
|
||||
case options.StorageTypeUnifiedGrpc:
|
||||
if opts.Address == "" {
|
||||
@@ -168,24 +171,14 @@ func newClient(opts options.StorageOptions,
|
||||
return resource.NewResourceClient(conn, indexConn, cfg, features, tracer)
|
||||
|
||||
default:
|
||||
// Create search options for the search server
|
||||
searchOptions, err := search.NewSearchOptions(features, cfg, docs, indexMetrics, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serverOptions := sql.ServerOptions{
|
||||
DB: db,
|
||||
Cfg: cfg,
|
||||
Tracer: tracer,
|
||||
Reg: reg,
|
||||
AccessClient: authzc,
|
||||
SearchOptions: searchOptions,
|
||||
StorageMetrics: storageMetrics,
|
||||
IndexMetrics: indexMetrics,
|
||||
Features: features,
|
||||
SecureValues: secure,
|
||||
}
|
||||
|
||||
// Setup QOS queue if enabled
|
||||
var qosQueue sql.QOSEnqueueDequeuer
|
||||
if cfg.QOSEnabled {
|
||||
qosReg := prometheus.WrapRegistererWithPrefix("resource_server_qos_", reg)
|
||||
queue := scheduler.NewQueue(&scheduler.QueueOptions{
|
||||
@@ -196,7 +189,7 @@ func newClient(opts options.StorageOptions,
|
||||
if err := services.StartAndAwaitRunning(ctx, queue); err != nil {
|
||||
return nil, fmt.Errorf("failed to start queue: %w", err)
|
||||
}
|
||||
scheduler, err := scheduler.NewScheduler(queue, &scheduler.Config{
|
||||
sched, err := scheduler.NewScheduler(queue, &scheduler.Config{
|
||||
NumWorkers: cfg.QOSNumberWorker,
|
||||
Logger: cfg.Logger,
|
||||
})
|
||||
@@ -204,31 +197,59 @@ func newClient(opts options.StorageOptions,
|
||||
return nil, fmt.Errorf("failed to create scheduler: %w", err)
|
||||
}
|
||||
|
||||
err = services.StartAndAwaitRunning(ctx, scheduler)
|
||||
err = services.StartAndAwaitRunning(ctx, sched)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start scheduler: %w", err)
|
||||
}
|
||||
serverOptions.QOSQueue = queue
|
||||
qosQueue = queue
|
||||
}
|
||||
|
||||
// only enable if an overrides file path is provided
|
||||
// Setup overrides service if enabled
|
||||
var overridesSvc *resource.OverridesService
|
||||
if cfg.OverridesFilePath != "" {
|
||||
overridesSvc, err := resource.NewOverridesService(ctx, cfg.Logger, reg, tracer, resource.ReloadOptions{
|
||||
overridesSvc, err = resource.NewOverridesService(ctx, cfg.Logger, reg, tracer, resource.ReloadOptions{
|
||||
FilePath: cfg.OverridesFilePath,
|
||||
ReloadPeriod: cfg.OverridesReloadInterval,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serverOptions.OverridesService = overridesSvc
|
||||
}
|
||||
|
||||
server, err := sql.NewResourceServer(serverOptions)
|
||||
// Create the search server with shared backend
|
||||
searchServer, err := sql.NewSearchServer(sql.SearchServerOptions{
|
||||
Backend: backend, // Use shared backend to avoid duplicate metrics registration
|
||||
DB: db,
|
||||
Cfg: cfg,
|
||||
Tracer: tracer,
|
||||
Reg: reg,
|
||||
AccessClient: authzc,
|
||||
SearchOptions: searchOptions,
|
||||
IndexMetrics: indexMetrics,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resource.NewLocalResourceClient(server), nil
|
||||
|
||||
// Create the storage server with shared backend
|
||||
storageServer, err := sql.NewStorageServer(sql.StorageServerOptions{
|
||||
Backend: backend, // Use shared backend to avoid duplicate metrics registration
|
||||
DB: db,
|
||||
Cfg: cfg,
|
||||
Tracer: tracer,
|
||||
Reg: reg,
|
||||
AccessClient: authzc,
|
||||
StorageMetrics: storageMetrics,
|
||||
Features: features,
|
||||
QOSQueue: qosQueue,
|
||||
SecureValues: secure,
|
||||
OverridesService: overridesSvc,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resource.NewLocalResourceClient(storageServer, searchServer), nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ func buildCollectionSettings(opts legacy.MigrateOptions) resource.BulkSettings {
|
||||
}
|
||||
|
||||
type resourceClientStreamProvider struct {
|
||||
client resource.ResourceClient
|
||||
client resource.MigratorClient
|
||||
}
|
||||
|
||||
func (r *resourceClientStreamProvider) createStream(ctx context.Context, opts legacy.MigrateOptions) (resourcepb.BulkStore_BulkProcessClient, error) {
|
||||
@@ -71,7 +71,7 @@ func (b *bulkStoreClientStreamProvider) createStream(ctx context.Context, opts l
|
||||
// This can migrate Folders, Dashboards and LibraryPanels
|
||||
func ProvideUnifiedMigrator(
|
||||
dashboardAccess legacy.MigrationDashboardAccessor,
|
||||
client resource.ResourceClient,
|
||||
client resource.MigratorClient,
|
||||
) UnifiedMigrator {
|
||||
return newUnifiedMigrator(
|
||||
dashboardAccess,
|
||||
|
||||
@@ -31,13 +31,31 @@ import (
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
//go:generate mockery --name ResourceClient --structname MockResourceClient --inpackage --filename client_mock.go --with-expecter
|
||||
type ResourceClient interface {
|
||||
resourcepb.ResourceStoreClient
|
||||
// SearchClient is used to interact with unified search
|
||||
type SearchClient interface {
|
||||
resourcepb.ResourceIndexClient
|
||||
resourcepb.ManagedObjectIndexClient
|
||||
resourcepb.BulkStoreClient
|
||||
}
|
||||
|
||||
// StorageClient is used to interact with unified storage
|
||||
type StorageClient interface {
|
||||
resourcepb.ResourceStoreClient
|
||||
resourcepb.BlobStoreClient
|
||||
}
|
||||
|
||||
// MigratorClient is used to perform migrations to unified storage
|
||||
type MigratorClient interface {
|
||||
resourcepb.BulkStoreClient
|
||||
GetStats(ctx context.Context, in *resourcepb.ResourceStatsRequest, opts ...grpc.CallOption) (*resourcepb.ResourceStatsResponse, error)
|
||||
}
|
||||
|
||||
// ResourceClient combines all resource-related clients and should be avoided in favor of more specific interfaces when possible
|
||||
//
|
||||
//go:generate mockery --name ResourceClient --structname MockResourceClient --inpackage --filename client_mock.go --with-expecter
|
||||
type ResourceClient interface {
|
||||
StorageClient
|
||||
SearchClient
|
||||
MigratorClient
|
||||
resourcepb.DiagnosticsClient
|
||||
resourcepb.QuotasClient
|
||||
}
|
||||
@@ -92,16 +110,15 @@ func NewLegacyResourceClient(channel grpc.ClientConnInterface, indexChannel grpc
|
||||
return newResourceClient(cc, cci)
|
||||
}
|
||||
|
||||
func NewLocalResourceClient(server ResourceServer) ResourceClient {
|
||||
func NewLocalResourceClient(server ResourceServer, searchServer SearchServer) ResourceClient {
|
||||
// scenario: local in-proc
|
||||
channel := &inprocgrpc.Channel{}
|
||||
indexChannel := &inprocgrpc.Channel{}
|
||||
tracer := otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/resource")
|
||||
|
||||
grpcAuthInt := grpcutils.NewUnsafeAuthenticator(tracer)
|
||||
for _, desc := range []*grpc.ServiceDesc{
|
||||
&resourcepb.ResourceStore_ServiceDesc,
|
||||
&resourcepb.ResourceIndex_ServiceDesc,
|
||||
&resourcepb.ManagedObjectIndex_ServiceDesc,
|
||||
&resourcepb.BlobStore_ServiceDesc,
|
||||
&resourcepb.BulkStore_ServiceDesc,
|
||||
&resourcepb.Diagnostics_ServiceDesc,
|
||||
@@ -117,13 +134,31 @@ func NewLocalResourceClient(server ResourceServer) ResourceClient {
|
||||
)
|
||||
}
|
||||
|
||||
// Register search services on the index channel if searchServer is provided
|
||||
if searchServer != nil {
|
||||
for _, desc := range []*grpc.ServiceDesc{
|
||||
&resourcepb.ResourceIndex_ServiceDesc,
|
||||
&resourcepb.ManagedObjectIndex_ServiceDesc,
|
||||
} {
|
||||
indexChannel.RegisterService(
|
||||
grpchan.InterceptServer(
|
||||
desc,
|
||||
grpcAuth.UnaryServerInterceptor(grpcAuthInt),
|
||||
grpcAuth.StreamServerInterceptor(grpcAuthInt),
|
||||
),
|
||||
searchServer,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
clientInt := authnlib.NewGrpcClientInterceptor(
|
||||
ProvideInProcExchanger(),
|
||||
authnlib.WithClientInterceptorIDTokenExtractor(idTokenExtractor),
|
||||
)
|
||||
|
||||
cc := grpchan.InterceptClientConn(channel, clientInt.UnaryClientInterceptor, clientInt.StreamClientInterceptor)
|
||||
return newResourceClient(cc, cc)
|
||||
cci := grpchan.InterceptClientConn(indexChannel, clientInt.UnaryClientInterceptor, clientInt.StreamClientInterceptor)
|
||||
return newResourceClient(cc, cci)
|
||||
}
|
||||
|
||||
type RemoteResourceClientConfig struct {
|
||||
|
||||
@@ -127,7 +127,10 @@ type SearchBackend interface {
|
||||
GetOpenIndexes() []NamespacedResource
|
||||
}
|
||||
|
||||
// This supports indexing+search regardless of implementation
|
||||
var _ SearchServer = &searchSupport{}
|
||||
|
||||
// This supports indexing+search regardless of implementation.
|
||||
// Implements SearchServer interface.
|
||||
type searchSupport struct {
|
||||
log log.Logger
|
||||
storage StorageBackend
|
||||
@@ -160,6 +163,10 @@ var (
|
||||
_ resourcepb.ManagedObjectIndexServer = (*searchSupport)(nil)
|
||||
)
|
||||
|
||||
func NewSearchServer(opts SearchOptions, storage StorageBackend, access types.AccessClient, blob BlobSupport, indexMetrics *BleveIndexMetrics, ownsIndexFn func(key NamespacedResource) (bool, error)) (SearchServer, error) {
|
||||
return newSearchSupport(opts, storage, access, blob, indexMetrics, ownsIndexFn)
|
||||
}
|
||||
|
||||
func newSearchSupport(opts SearchOptions, storage StorageBackend, access types.AccessClient, blob BlobSupport, indexMetrics *BleveIndexMetrics, ownsIndexFn func(key NamespacedResource) (bool, error)) (support *searchSupport, err error) {
|
||||
// No backend search support
|
||||
if opts.Backend == nil {
|
||||
@@ -598,6 +605,22 @@ func (s *searchSupport) buildIndexes(ctx context.Context) (int, error) {
|
||||
return totalBatchesIndexed, nil
|
||||
}
|
||||
|
||||
func (s *searchSupport) Init(ctx context.Context) error {
|
||||
return s.init(ctx)
|
||||
}
|
||||
|
||||
func (s *searchSupport) Stop(_ context.Context) error {
|
||||
s.stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsHealthy implements resourcepb.DiagnosticsServer
|
||||
func (s *searchSupport) IsHealthy(ctx context.Context, req *resourcepb.HealthCheckRequest) (*resourcepb.HealthCheckResponse, error) {
|
||||
return &resourcepb.HealthCheckResponse{
|
||||
Status: resourcepb.HealthCheckResponse_SERVING,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *searchSupport) init(ctx context.Context) error {
|
||||
origCtx := ctx
|
||||
|
||||
@@ -863,7 +886,7 @@ func newRebuildRequest(key NamespacedResource, minBuildTime, lastImportTime time
|
||||
|
||||
func (s *searchSupport) getOrCreateIndex(ctx context.Context, stats *SearchStats, key NamespacedResource, reason string) (ResourceIndex, error) {
|
||||
if s == nil || s.search == nil {
|
||||
return nil, fmt.Errorf("search is not configured properly (missing unifiedStorageSearch feature toggle?)")
|
||||
return nil, fmt.Errorf("search is not configured properly (missing enable_search config?)")
|
||||
}
|
||||
|
||||
ctx, span := tracer.Start(ctx, "resource.searchSupport.getOrCreateIndex")
|
||||
|
||||
@@ -60,7 +60,7 @@ func ProvideSearchDistributorServer(cfg *setting.Cfg, features featuremgmt.Featu
|
||||
}
|
||||
|
||||
type RingClient struct {
|
||||
Client ResourceClient
|
||||
Client SearchClient
|
||||
grpc_health_v1.HealthClient
|
||||
Conn *grpc.ClientConn
|
||||
}
|
||||
@@ -99,7 +99,7 @@ var (
|
||||
func (ds *distributorServer) Search(ctx context.Context, r *resourcepb.ResourceSearchRequest) (*resourcepb.ResourceSearchResponse, error) {
|
||||
ctx, span := ds.tracing.Start(ctx, "distributor.Search")
|
||||
defer span.End()
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Options.Key.Namespace, "Search")
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Options.Key.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func (ds *distributorServer) Search(ctx context.Context, r *resourcepb.ResourceS
|
||||
func (ds *distributorServer) GetStats(ctx context.Context, r *resourcepb.ResourceStatsRequest) (*resourcepb.ResourceStatsResponse, error) {
|
||||
ctx, span := ds.tracing.Start(ctx, "distributor.GetStats")
|
||||
defer span.End()
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Namespace, "GetStats")
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -215,7 +215,7 @@ func (ds *distributorServer) RebuildIndexes(ctx context.Context, r *resourcepb.R
|
||||
func (ds *distributorServer) CountManagedObjects(ctx context.Context, r *resourcepb.CountManagedObjectsRequest) (*resourcepb.CountManagedObjectsResponse, error) {
|
||||
ctx, span := ds.tracing.Start(ctx, "distributor.CountManagedObjects")
|
||||
defer span.End()
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Namespace, "CountManagedObjects")
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -226,7 +226,7 @@ func (ds *distributorServer) CountManagedObjects(ctx context.Context, r *resourc
|
||||
func (ds *distributorServer) ListManagedObjects(ctx context.Context, r *resourcepb.ListManagedObjectsRequest) (*resourcepb.ListManagedObjectsResponse, error) {
|
||||
ctx, span := ds.tracing.Start(ctx, "distributor.ListManagedObjects")
|
||||
defer span.End()
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Namespace, "ListManagedObjects")
|
||||
ctx, client, err := ds.getClientToDistributeRequest(ctx, r.Namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -234,7 +234,7 @@ func (ds *distributorServer) ListManagedObjects(ctx context.Context, r *resource
|
||||
return client.ListManagedObjects(ctx, r)
|
||||
}
|
||||
|
||||
func (ds *distributorServer) getClientToDistributeRequest(ctx context.Context, namespace string, methodName string) (context.Context, ResourceClient, error) {
|
||||
func (ds *distributorServer) getClientToDistributeRequest(ctx context.Context, namespace string) (context.Context, SearchClient, error) {
|
||||
ringHasher := fnv.New32a()
|
||||
_, err := ringHasher.Write([]byte(namespace))
|
||||
if err != nil {
|
||||
|
||||
@@ -34,12 +34,18 @@ import (
|
||||
|
||||
var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/resource")
|
||||
|
||||
type SearchServer interface {
|
||||
LifecycleHooks
|
||||
|
||||
resourcepb.ResourceIndexServer
|
||||
resourcepb.ManagedObjectIndexServer
|
||||
resourcepb.DiagnosticsServer
|
||||
}
|
||||
|
||||
// ResourceServer implements all gRPC services
|
||||
type ResourceServer interface {
|
||||
resourcepb.ResourceStoreServer
|
||||
resourcepb.BulkStoreServer
|
||||
resourcepb.ResourceIndexServer
|
||||
resourcepb.ManagedObjectIndexServer
|
||||
resourcepb.BlobStoreServer
|
||||
resourcepb.DiagnosticsServer
|
||||
resourcepb.QuotasServer
|
||||
@@ -221,9 +227,6 @@ type ResourceServerOptions struct {
|
||||
// The blob configuration
|
||||
Blob BlobConfig
|
||||
|
||||
// Search options
|
||||
Search SearchOptions
|
||||
|
||||
// Quota service
|
||||
OverridesService *OverridesService
|
||||
|
||||
@@ -251,16 +254,15 @@ type ResourceServerOptions struct {
|
||||
|
||||
storageMetrics *StorageMetrics
|
||||
|
||||
IndexMetrics *BleveIndexMetrics
|
||||
|
||||
// MaxPageSizeBytes is the maximum size of a page in bytes.
|
||||
MaxPageSizeBytes int
|
||||
// IndexMinUpdateInterval is the time to wait after a successful write operation to ensure read-after-write consistency in search.
|
||||
// This config is shared with search
|
||||
IndexMinUpdateInterval time.Duration
|
||||
|
||||
// QOSQueue is the quality of service queue used to enqueue
|
||||
QOSQueue QOSEnqueuer
|
||||
QOSConfig QueueConfig
|
||||
|
||||
OwnsIndexFn func(key NamespacedResource) (bool, error)
|
||||
}
|
||||
|
||||
func NewResourceServer(opts ResourceServerOptions) (*server, error) {
|
||||
@@ -343,23 +345,24 @@ func NewResourceServer(opts ResourceServerOptions) (*server, error) {
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
storageMetrics: opts.storageMetrics,
|
||||
indexMetrics: opts.IndexMetrics,
|
||||
maxPageSizeBytes: opts.MaxPageSizeBytes,
|
||||
reg: opts.Reg,
|
||||
queue: opts.QOSQueue,
|
||||
queueConfig: opts.QOSConfig,
|
||||
overridesService: opts.OverridesService,
|
||||
|
||||
artificialSuccessfulWriteDelay: opts.Search.IndexMinUpdateInterval,
|
||||
artificialSuccessfulWriteDelay: opts.IndexMinUpdateInterval,
|
||||
}
|
||||
|
||||
if opts.Search.Resources != nil {
|
||||
var err error
|
||||
s.search, err = newSearchSupport(opts.Search, s.backend, s.access, s.blob, opts.IndexMetrics, opts.OwnsIndexFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
/*
|
||||
if opts.Search.Resources != nil {
|
||||
var err error
|
||||
s.search, err = newSearchSupport(opts.Search, s.backend, s.access, s.blob, opts.IndexMetrics, opts.OwnsIndexFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
err := s.Init(ctx)
|
||||
if err != nil {
|
||||
@@ -377,7 +380,6 @@ type server struct {
|
||||
backend StorageBackend
|
||||
blob BlobSupport
|
||||
secure secrets.InlineSecureValueSupport
|
||||
search *searchSupport
|
||||
diagnostics resourcepb.DiagnosticsServer
|
||||
access claims.AccessClient
|
||||
writeHooks WriteAccessHooks
|
||||
@@ -424,11 +426,6 @@ func (s *server) Init(ctx context.Context) error {
|
||||
s.initErr = s.overridesService.init(ctx)
|
||||
}
|
||||
|
||||
// initialize the search index
|
||||
if s.initErr == nil && s.search != nil {
|
||||
s.initErr = s.search.init(ctx)
|
||||
}
|
||||
|
||||
// Start watching for changes
|
||||
if s.initErr == nil {
|
||||
s.initErr = s.initWatcher()
|
||||
@@ -453,10 +450,6 @@ func (s *server) Stop(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
if s.search != nil {
|
||||
s.search.stop()
|
||||
}
|
||||
|
||||
if s.overridesService != nil {
|
||||
if err := s.overridesService.stop(ctx); err != nil {
|
||||
stopFailed = true
|
||||
@@ -1372,47 +1365,6 @@ func (s *server) Watch(req *resourcepb.WatchRequest, srv resourcepb.ResourceStor
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) Search(ctx context.Context, req *resourcepb.ResourceSearchRequest) (*resourcepb.ResourceSearchResponse, error) {
|
||||
if s.search == nil {
|
||||
return nil, fmt.Errorf("search index not configured")
|
||||
}
|
||||
|
||||
return s.search.Search(ctx, req)
|
||||
}
|
||||
|
||||
// GetStats implements ResourceServer.
|
||||
func (s *server) GetStats(ctx context.Context, req *resourcepb.ResourceStatsRequest) (*resourcepb.ResourceStatsResponse, error) {
|
||||
if err := s.Init(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if s.search == nil {
|
||||
// If the backend implements "GetStats", we can use it
|
||||
srv, ok := s.backend.(resourcepb.ResourceIndexServer)
|
||||
if ok {
|
||||
return srv.GetStats(ctx, req)
|
||||
}
|
||||
return nil, fmt.Errorf("search index not configured")
|
||||
}
|
||||
return s.search.GetStats(ctx, req)
|
||||
}
|
||||
|
||||
func (s *server) ListManagedObjects(ctx context.Context, req *resourcepb.ListManagedObjectsRequest) (*resourcepb.ListManagedObjectsResponse, error) {
|
||||
if s.search == nil {
|
||||
return nil, fmt.Errorf("search index not configured")
|
||||
}
|
||||
|
||||
return s.search.ListManagedObjects(ctx, req)
|
||||
}
|
||||
|
||||
func (s *server) CountManagedObjects(ctx context.Context, req *resourcepb.CountManagedObjectsRequest) (*resourcepb.CountManagedObjectsResponse, error) {
|
||||
if s.search == nil {
|
||||
return nil, fmt.Errorf("search index not configured")
|
||||
}
|
||||
|
||||
return s.search.CountManagedObjects(ctx, req)
|
||||
}
|
||||
|
||||
// IsHealthy implements ResourceServer.
|
||||
func (s *server) IsHealthy(ctx context.Context, req *resourcepb.HealthCheckRequest) (*resourcepb.HealthCheckResponse, error) {
|
||||
return s.diagnostics.IsHealthy(ctx, req)
|
||||
@@ -1568,14 +1520,6 @@ func (s *server) runInQueue(ctx context.Context, tenantID string, runnable func(
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) RebuildIndexes(ctx context.Context, req *resourcepb.RebuildIndexesRequest) (*resourcepb.RebuildIndexesResponse, error) {
|
||||
if s.search == nil {
|
||||
return nil, fmt.Errorf("search index not configured")
|
||||
}
|
||||
|
||||
return s.search.RebuildIndexes(ctx, req)
|
||||
}
|
||||
|
||||
func (s *server) checkQuota(ctx context.Context, nsr NamespacedResource) {
|
||||
span := trace.SpanFromContext(ctx)
|
||||
span.AddEvent("checkQuota", trace.WithAttributes(
|
||||
|
||||
@@ -1253,21 +1253,23 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R
|
||||
queryExact.SetField(resource.SEARCH_FIELD_TITLE)
|
||||
queryExact.Analyzer = keyword.Name // don't analyze the query input - treat it as a single token
|
||||
queryExact.Operator = query.MatchQueryOperatorAnd // This doesn't make a difference for keyword analyzer, we add it just to be explicit.
|
||||
searchQuery := bleve.NewDisjunctionQuery(queryExact)
|
||||
|
||||
// Query 2: Phrase query with standard analyzer
|
||||
queryPhrase := bleve.NewMatchPhraseQuery(req.Query)
|
||||
queryPhrase.SetBoost(5.0)
|
||||
queryPhrase.SetField(resource.SEARCH_FIELD_TITLE)
|
||||
queryPhrase.Analyzer = standard.Name
|
||||
searchQuery.AddQuery(queryPhrase)
|
||||
|
||||
// Query 3: Match query with standard analyzer
|
||||
queryAnalyzed := bleve.NewMatchQuery(removeSmallTerms(req.Query))
|
||||
queryAnalyzed.SetField(resource.SEARCH_FIELD_TITLE)
|
||||
queryAnalyzed.SetBoost(2.0)
|
||||
queryAnalyzed.Analyzer = standard.Name
|
||||
queryAnalyzed.Operator = query.MatchQueryOperatorAnd // Make sure all terms from the query are matched
|
||||
searchQuery.AddQuery(queryAnalyzed)
|
||||
|
||||
// At least one of the queries must match
|
||||
searchQuery := bleve.NewDisjunctionQuery(queryExact, queryAnalyzed, queryPhrase)
|
||||
queries = append(queries, searchQuery)
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"go.uber.org/goleak"
|
||||
|
||||
authlib "github.com/grafana/authlib/types"
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
const DASHBOARD_SCHEMA_VERSION = "schema_version"
|
||||
const DASHBOARD_LINK_COUNT = "link_count"
|
||||
const DASHBOARD_PANEL_TYPES = "panel_types"
|
||||
const DASHBOARD_PANEL_TITLE = "panel_title"
|
||||
const DASHBOARD_DS_TYPES = "ds_types"
|
||||
const DASHBOARD_TRANSFORMATIONS = "transformation"
|
||||
const DASHBOARD_LIBRARY_PANEL_REFERENCE = "reference.LibraryPanel"
|
||||
@@ -53,11 +55,21 @@ func DashboardBuilder(namespaced resource.NamespacedDocumentSupplier) (resource.
|
||||
Type: resourcepb.ResourceTableColumnDefinition_INT32,
|
||||
Description: "How many links appear on the page",
|
||||
},
|
||||
{
|
||||
Name: DASHBOARD_PANEL_TITLE,
|
||||
Type: resourcepb.ResourceTableColumnDefinition_STRING,
|
||||
IsArray: true,
|
||||
Description: "The panel title text",
|
||||
Properties: &resourcepb.ResourceTableColumnDefinition_Properties{
|
||||
Filterable: false, // full text
|
||||
FreeText: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: DASHBOARD_PANEL_TYPES,
|
||||
Type: resourcepb.ResourceTableColumnDefinition_STRING,
|
||||
IsArray: true,
|
||||
Description: "How many links appear on the page",
|
||||
Description: "The panel types used in this dashboard",
|
||||
Properties: &resourcepb.ResourceTableColumnDefinition_Properties{
|
||||
Filterable: true,
|
||||
},
|
||||
@@ -269,14 +281,22 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
|
||||
doc.Description = summary.Description
|
||||
doc.Tags = summary.Tags
|
||||
|
||||
panelTitles := []string{}
|
||||
panelTypes := []string{}
|
||||
transformations := []string{}
|
||||
dsTypes := []string{}
|
||||
|
||||
for _, p := range summary.Panels {
|
||||
if p.Type != "" {
|
||||
for p := range summary.PanelIterator() {
|
||||
switch p.Type {
|
||||
case "": // ignore
|
||||
case "row": // row should map to a layout type when we support v2 constructs
|
||||
default:
|
||||
panelTypes = append(panelTypes, p.Type)
|
||||
}
|
||||
|
||||
if len(p.Title) > 0 {
|
||||
panelTitles = append(panelTitles, p.Title)
|
||||
}
|
||||
if len(p.Transformer) > 0 {
|
||||
transformations = append(transformations, p.Transformer...)
|
||||
}
|
||||
@@ -309,17 +329,20 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
|
||||
resource.SEARCH_FIELD_LEGACY_ID: summary.ID,
|
||||
}
|
||||
|
||||
if len(panelTitles) > 0 {
|
||||
doc.Fields[DASHBOARD_PANEL_TITLE] = panelTitles
|
||||
}
|
||||
if len(panelTypes) > 0 {
|
||||
sort.Strings(panelTypes)
|
||||
doc.Fields[DASHBOARD_PANEL_TYPES] = panelTypes
|
||||
doc.Fields[DASHBOARD_PANEL_TYPES] = slices.Compact(panelTypes) // distinct values
|
||||
}
|
||||
if len(dsTypes) > 0 {
|
||||
sort.Strings(dsTypes)
|
||||
doc.Fields[DASHBOARD_DS_TYPES] = dsTypes
|
||||
doc.Fields[DASHBOARD_DS_TYPES] = slices.Compact(dsTypes) // distinct values
|
||||
}
|
||||
if len(transformations) > 0 {
|
||||
sort.Strings(transformations)
|
||||
doc.Fields[DASHBOARD_TRANSFORMATIONS] = transformations
|
||||
doc.Fields[DASHBOARD_TRANSFORMATIONS] = slices.Compact(transformations) // distinct values
|
||||
}
|
||||
|
||||
for k, v := range s.Stats[summary.UID] {
|
||||
|
||||
@@ -32,10 +32,16 @@
|
||||
"errors_last_7_days": 1,
|
||||
"grafana.app/deprecatedInternalID": 141,
|
||||
"link_count": 0,
|
||||
"panel_title": [
|
||||
"green pie",
|
||||
"red pie",
|
||||
"blue pie",
|
||||
"collapsed row"
|
||||
],
|
||||
"panel_types": [
|
||||
"barchart",
|
||||
"graph",
|
||||
"row"
|
||||
"pie"
|
||||
],
|
||||
"schema_version": 38
|
||||
},
|
||||
@@ -46,6 +52,12 @@
|
||||
"kind": "DataSource",
|
||||
"name": "DSUID"
|
||||
},
|
||||
{
|
||||
"relation": "depends-on",
|
||||
"group": "dashboards.grafana.app",
|
||||
"kind": "LibraryPanel",
|
||||
"name": "l3d2s634-fdgf-75u4-3fg3-67j966ii7jur"
|
||||
},
|
||||
{
|
||||
"relation": "depends-on",
|
||||
"group": "dashboards.grafana.app",
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
"name": "red pie",
|
||||
"uid": "e1d5f519-dabd-47c6-9ad7-83d181ce1cee"
|
||||
},
|
||||
"title": "green pie"
|
||||
"title": "red pie"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
@@ -78,6 +78,14 @@
|
||||
"id": 8,
|
||||
"type": "graph"
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "graph"
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "graph"
|
||||
},
|
||||
{
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
@@ -101,6 +109,10 @@
|
||||
"uid": "l3d2s634-fdgf-75u4-3fg3-67j966ii7jur"
|
||||
},
|
||||
"title": "blue pie"
|
||||
},
|
||||
{
|
||||
"id": 40,
|
||||
"type": "pie"
|
||||
}
|
||||
],
|
||||
"title": "collapsed row",
|
||||
|
||||
@@ -19,7 +19,7 @@ func NewSearchOptions(
|
||||
ownsIndexFn func(key resource.NamespacedResource) (bool, error),
|
||||
) (resource.SearchOptions, error) {
|
||||
//nolint:staticcheck // not yet migrated to OpenFeature
|
||||
if cfg.EnableSearch || features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageSearch) || features.IsEnabledGlobally(featuremgmt.FlagProvisioning) {
|
||||
if cfg.EnableSearch || features.IsEnabledGlobally(featuremgmt.FlagProvisioning) {
|
||||
root := cfg.IndexPath
|
||||
if root == "" {
|
||||
root = filepath.Join(cfg.DataPath, "unified-search", "bleve")
|
||||
|
||||
+11
-1
@@ -71,11 +71,18 @@
|
||||
"description": "How many links appear on the page",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "panel_title",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "The panel title text",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
"name": "panel_types",
|
||||
"type": "string",
|
||||
"format": "",
|
||||
"description": "How many links appear on the page",
|
||||
"description": "The panel types used in this dashboard",
|
||||
"priority": 0
|
||||
},
|
||||
{
|
||||
@@ -214,6 +221,7 @@
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
],
|
||||
"object": {
|
||||
@@ -239,6 +247,7 @@
|
||||
"repo",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
[
|
||||
"timeseries"
|
||||
],
|
||||
@@ -282,6 +291,7 @@
|
||||
"repo",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
[
|
||||
"timeseries",
|
||||
"table"
|
||||
|
||||
@@ -11,6 +11,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
infraDB "github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/lib/pq"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -44,10 +47,63 @@ const defaultPrunerHistoryLimit = 20
|
||||
|
||||
func ProvideStorageBackend(
|
||||
cfg *setting.Cfg,
|
||||
db infraDB.DB,
|
||||
tracer trace.Tracer,
|
||||
reg prometheus.Registerer,
|
||||
storageMetrics *resource.StorageMetrics,
|
||||
) (resource.StorageBackend, error) {
|
||||
// TODO: make this the central place to provide SQL backend
|
||||
// Currently it is skipped as we need to handle the cases of Diagnostics and Lifecycle
|
||||
return nil, nil
|
||||
// Create the resource DB
|
||||
eDB, err := dbimpl.ProvideResourceDB(db, cfg, tracer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create resource DB: %w", err)
|
||||
}
|
||||
|
||||
// Check if HA is enabled
|
||||
isHA := isHighAvailabilityEnabled(
|
||||
cfg.SectionWithEnvOverrides("database"),
|
||||
cfg.SectionWithEnvOverrides("resource_api"),
|
||||
)
|
||||
|
||||
// Create the backend
|
||||
backend, err := NewBackend(BackendOptions{
|
||||
DBProvider: eDB,
|
||||
Reg: reg,
|
||||
IsHA: isHA,
|
||||
storageMetrics: storageMetrics,
|
||||
LastImportTimeMaxAge: cfg.MaxFileIndexAge,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create backend: %w", err)
|
||||
}
|
||||
|
||||
// Initialize the backend
|
||||
if err := backend.Init(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize backend: %w", err)
|
||||
}
|
||||
|
||||
return backend, nil
|
||||
}
|
||||
|
||||
// isHighAvailabilityEnabled determines if high availability mode should
|
||||
// be enabled based on database configuration. High availability is enabled
|
||||
// by default except for SQLite databases.
|
||||
func isHighAvailabilityEnabled(dbCfg, resourceAPICfg *setting.DynamicSection) bool {
|
||||
// If the resource API is using a non-SQLite database, we assume it's in HA mode.
|
||||
resourceDBType := resourceAPICfg.Key("db_type").String()
|
||||
if resourceDBType != "" && resourceDBType != migrator.SQLite {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check in the config if HA is enabled - by default we always assume a HA setup.
|
||||
isHA := dbCfg.Key("high_availability").MustBool(true)
|
||||
|
||||
// SQLite is not possible to run in HA, so we force it to false.
|
||||
databaseType := dbCfg.Key("type").String()
|
||||
if databaseType == migrator.SQLite {
|
||||
isHA = false
|
||||
}
|
||||
|
||||
return isHA
|
||||
}
|
||||
|
||||
type Backend interface {
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"google.golang.org/grpc/health/grpc_health_v1"
|
||||
|
||||
"github.com/grafana/dskit/kv"
|
||||
"github.com/grafana/dskit/ring"
|
||||
"github.com/grafana/dskit/services"
|
||||
|
||||
infraDB "github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/modules"
|
||||
"github.com/grafana/grafana/pkg/services/authz"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/grpcserver"
|
||||
"github.com/grafana/grafana/pkg/services/grpcserver/interceptors"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
)
|
||||
|
||||
var _ UnifiedStorageGrpcService = (*searchService)(nil)
|
||||
|
||||
// operation used by the search-servers to check if they own the namespace
|
||||
var (
|
||||
searchOwnerRead = ring.NewOp([]ring.InstanceState{ring.JOINING, ring.ACTIVE, ring.LEAVING}, nil)
|
||||
)
|
||||
|
||||
type searchService struct {
|
||||
*services.BasicService
|
||||
|
||||
backend resource.StorageBackend
|
||||
cfg *setting.Cfg
|
||||
features featuremgmt.FeatureToggles
|
||||
db infraDB.DB
|
||||
stopCh chan struct{}
|
||||
stoppedCh chan error
|
||||
handler grpcserver.Provider
|
||||
tracing trace.Tracer
|
||||
authenticator func(ctx context.Context) (context.Context, error)
|
||||
httpServerRouter *mux.Router
|
||||
|
||||
log log.Logger
|
||||
reg prometheus.Registerer
|
||||
docBuilders resource.DocumentBuilderSupplier
|
||||
indexMetrics *resource.BleveIndexMetrics
|
||||
searchRing *ring.Ring
|
||||
|
||||
// Ring lifecycle and sharding support
|
||||
ringLifecycler *ring.BasicLifecycler
|
||||
|
||||
// Subservices manager
|
||||
subservices *services.Manager
|
||||
subservicesWatcher *services.FailureWatcher
|
||||
hasSubservices bool
|
||||
}
|
||||
|
||||
func ProvideUnifiedSearchGrpcService(
|
||||
cfg *setting.Cfg,
|
||||
features featuremgmt.FeatureToggles,
|
||||
db infraDB.DB,
|
||||
log log.Logger,
|
||||
reg prometheus.Registerer,
|
||||
docBuilders resource.DocumentBuilderSupplier,
|
||||
indexMetrics *resource.BleveIndexMetrics,
|
||||
searchRing *ring.Ring,
|
||||
memberlistKVConfig kv.Config,
|
||||
backend resource.StorageBackend,
|
||||
httpServerRouter *mux.Router,
|
||||
) (UnifiedStorageGrpcService, error) {
|
||||
var err error
|
||||
tracer := otel.Tracer("unified-search-server")
|
||||
|
||||
authn := NewAuthenticatorWithFallback(cfg, reg, tracer, func(ctx context.Context) (context.Context, error) {
|
||||
auth := grpc.Authenticator{Tracer: tracer}
|
||||
return auth.Authenticate(ctx)
|
||||
})
|
||||
|
||||
s := &searchService{
|
||||
backend: backend,
|
||||
cfg: cfg,
|
||||
features: features,
|
||||
stopCh: make(chan struct{}),
|
||||
stoppedCh: make(chan error, 1),
|
||||
authenticator: authn,
|
||||
tracing: tracer,
|
||||
db: db,
|
||||
log: log,
|
||||
reg: reg,
|
||||
docBuilders: docBuilders,
|
||||
indexMetrics: indexMetrics,
|
||||
searchRing: searchRing,
|
||||
httpServerRouter: httpServerRouter,
|
||||
subservicesWatcher: services.NewFailureWatcher(),
|
||||
}
|
||||
|
||||
subservices := []services.Service{}
|
||||
if cfg.EnableSharding {
|
||||
ringStore, err := kv.NewClient(
|
||||
memberlistKVConfig,
|
||||
ring.GetCodec(),
|
||||
kv.RegistererWithKVName(reg, resource.RingName),
|
||||
log,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create KV store client: %s", err)
|
||||
}
|
||||
|
||||
lifecyclerCfg, err := toLifecyclerConfig(cfg, log)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize search-ring lifecycler config: %s", err)
|
||||
}
|
||||
|
||||
// Define lifecycler delegates in reverse order (last to be called defined first because they're
|
||||
// chained via "next delegate").
|
||||
delegate := ring.BasicLifecyclerDelegate(ring.NewInstanceRegisterDelegate(ring.JOINING, resource.RingNumTokens))
|
||||
delegate = ring.NewLeaveOnStoppingDelegate(delegate, log)
|
||||
delegate = ring.NewAutoForgetDelegate(resource.RingHeartbeatTimeout*2, delegate, log)
|
||||
|
||||
s.ringLifecycler, err = ring.NewBasicLifecycler(
|
||||
lifecyclerCfg,
|
||||
resource.RingName,
|
||||
resource.RingKey,
|
||||
ringStore,
|
||||
delegate,
|
||||
log,
|
||||
reg,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize search-ring lifecycler: %s", err)
|
||||
}
|
||||
|
||||
s.ringLifecycler.SetKeepInstanceInTheRingOnShutdown(true)
|
||||
subservices = append(subservices, s.ringLifecycler)
|
||||
}
|
||||
|
||||
if len(subservices) > 0 {
|
||||
s.hasSubservices = true
|
||||
s.subservices, err = services.NewManager(subservices...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create subservices manager: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// This will be used when running as a dskit service
|
||||
s.BasicService = services.NewBasicService(s.starting, s.running, s.stopping).WithName(modules.SearchServer)
|
||||
|
||||
// Register HTTP endpoints if router is provided
|
||||
s.RegisterHTTPEndpoints(httpServerRouter)
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *searchService) PrepareDownscale(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
s.log.Info("Preparing for downscale. Will not keep instance in ring on shutdown.")
|
||||
s.ringLifecycler.SetKeepInstanceInTheRingOnShutdown(false)
|
||||
case http.MethodDelete:
|
||||
s.log.Info("Downscale canceled. Will keep instance in ring on shutdown.")
|
||||
s.ringLifecycler.SetKeepInstanceInTheRingOnShutdown(true)
|
||||
case http.MethodGet:
|
||||
// used for delayed downscale use case, which we don't support. Leaving here for completion sake
|
||||
s.log.Info("Received GET request for prepare-downscale. Behavior not implemented.")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *searchService) OwnsIndex(key resource.NamespacedResource) (bool, error) {
|
||||
if s.searchRing == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if st := s.searchRing.State(); st != services.Running {
|
||||
return false, fmt.Errorf("ring is not Running: %s", st)
|
||||
}
|
||||
|
||||
ringHasher := fnv.New32a()
|
||||
_, err := ringHasher.Write([]byte(key.Namespace))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error hashing namespace: %w", err)
|
||||
}
|
||||
|
||||
rs, err := s.searchRing.GetWithOptions(ringHasher.Sum32(), searchOwnerRead, ring.WithReplicationFactor(s.searchRing.ReplicationFactor()))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error getting replicaset from ring: %w", err)
|
||||
}
|
||||
|
||||
return rs.Includes(s.ringLifecycler.GetInstanceAddr()), nil
|
||||
}
|
||||
|
||||
func (s *searchService) starting(ctx context.Context) error {
|
||||
if s.hasSubservices {
|
||||
s.subservicesWatcher.WatchManager(s.subservices)
|
||||
if err := services.StartManagerAndAwaitHealthy(ctx, s.subservices); err != nil {
|
||||
return fmt.Errorf("failed to start subservices: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
authzClient, err := authz.ProvideStandaloneAuthZClient(s.cfg, s.features, s.tracing, s.reg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create search options for the search server
|
||||
searchOptions, err := search.NewSearchOptions(s.features, s.cfg, s.docBuilders, s.indexMetrics, s.OwnsIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the search server
|
||||
searchServer, err := NewSearchServer(SearchServerOptions{
|
||||
Backend: s.backend,
|
||||
DB: s.db,
|
||||
Cfg: s.cfg,
|
||||
Tracer: s.tracing,
|
||||
Reg: s.reg,
|
||||
AccessClient: authzClient,
|
||||
SearchOptions: searchOptions,
|
||||
IndexMetrics: s.indexMetrics,
|
||||
OwnsIndexFn: s.OwnsIndex,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.handler, err = grpcserver.ProvideService(s.cfg, s.features, interceptors.AuthenticatorFunc(s.authenticator), s.tracing, prometheus.DefaultRegisterer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
healthService, err := resource.ProvideHealthService(searchServer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srv := s.handler.GetServer()
|
||||
// Register search services
|
||||
resourcepb.RegisterResourceIndexServer(srv, searchServer)
|
||||
resourcepb.RegisterManagedObjectIndexServer(srv, searchServer)
|
||||
resourcepb.RegisterDiagnosticsServer(srv, searchServer)
|
||||
grpc_health_v1.RegisterHealthServer(srv, healthService)
|
||||
|
||||
// register reflection service
|
||||
_, err = grpcserver.ProvideReflectionService(s.cfg, s.handler)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.cfg.EnableSharding {
|
||||
s.log.Info("waiting until search server is JOINING in the ring")
|
||||
lfcCtx, cancel := context.WithTimeout(context.Background(), s.cfg.ResourceServerJoinRingTimeout)
|
||||
defer cancel()
|
||||
if err := ring.WaitInstanceState(lfcCtx, s.searchRing, s.ringLifecycler.GetInstanceID(), ring.JOINING); err != nil {
|
||||
return fmt.Errorf("error switching to JOINING in the ring: %s", err)
|
||||
}
|
||||
s.log.Info("search server is JOINING in the ring")
|
||||
|
||||
if err := s.ringLifecycler.ChangeState(ctx, ring.ACTIVE); err != nil {
|
||||
return fmt.Errorf("error switching to ACTIVE in the ring: %s", err)
|
||||
}
|
||||
s.log.Info("search server is ACTIVE in the ring")
|
||||
}
|
||||
|
||||
// start the gRPC server
|
||||
go func() {
|
||||
err := s.handler.Run(ctx)
|
||||
if err != nil {
|
||||
s.stoppedCh <- err
|
||||
} else {
|
||||
s.stoppedCh <- nil
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAddress returns the address of the gRPC server.
|
||||
func (s *searchService) GetAddress() string {
|
||||
return s.handler.GetAddress()
|
||||
}
|
||||
|
||||
func (s *searchService) running(ctx context.Context) error {
|
||||
select {
|
||||
case err := <-s.stoppedCh:
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
case err := <-s.subservicesWatcher.Chan():
|
||||
return fmt.Errorf("subservice failure: %w", err)
|
||||
case <-ctx.Done():
|
||||
close(s.stopCh)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *searchService) stopping(_ error) error {
|
||||
if s.hasSubservices {
|
||||
err := services.StopManagerAndAwaitStopped(context.Background(), s.subservices)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stop subservices: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *searchService) RegisterHTTPEndpoints(httpServerRouter *mux.Router) {
|
||||
if httpServerRouter != nil && s.cfg.EnableSharding {
|
||||
httpServerRouter.Path("/prepare-downscale").Methods("GET", "POST", "DELETE").Handler(http.HandlerFunc(s.PrepareDownscale))
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
secrets "github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
|
||||
inlinesecurevalue "github.com/grafana/grafana/pkg/registry/apis/secret/inline"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
@@ -30,8 +29,21 @@ type QOSEnqueueDequeuer interface {
|
||||
Dequeue(ctx context.Context) (func(), error)
|
||||
}
|
||||
|
||||
// ServerOptions contains the options for creating a new ResourceServer
|
||||
type ServerOptions struct {
|
||||
// SearchServerOptions contains the options for creating a new SearchServer
|
||||
type SearchServerOptions struct {
|
||||
Backend resource.StorageBackend
|
||||
DB infraDB.DB
|
||||
Cfg *setting.Cfg
|
||||
Tracer trace.Tracer
|
||||
Reg prometheus.Registerer
|
||||
AccessClient types.AccessClient
|
||||
SearchOptions resource.SearchOptions
|
||||
IndexMetrics *resource.BleveIndexMetrics
|
||||
OwnsIndexFn func(key resource.NamespacedResource) (bool, error)
|
||||
}
|
||||
|
||||
// StorageServerOptions contains the options for creating a storage-only server (without search)
|
||||
type StorageServerOptions struct {
|
||||
Backend resource.StorageBackend
|
||||
OverridesService *resource.OverridesService
|
||||
DB infraDB.DB
|
||||
@@ -39,16 +51,66 @@ type ServerOptions struct {
|
||||
Tracer trace.Tracer
|
||||
Reg prometheus.Registerer
|
||||
AccessClient types.AccessClient
|
||||
SearchOptions resource.SearchOptions
|
||||
StorageMetrics *resource.StorageMetrics
|
||||
IndexMetrics *resource.BleveIndexMetrics
|
||||
Features featuremgmt.FeatureToggles
|
||||
QOSQueue QOSEnqueueDequeuer
|
||||
SecureValues secrets.InlineSecureValueSupport
|
||||
OwnsIndexFn func(key resource.NamespacedResource) (bool, error)
|
||||
}
|
||||
|
||||
func NewResourceServer(opts ServerOptions) (resource.ResourceServer, error) {
|
||||
// NewSearchServer creates a new SearchServer with the given options.
|
||||
// This can be used to create a standalone search server or to create a search server
|
||||
// that will be passed to NewResourceServer.
|
||||
//
|
||||
// Important: When running in monolith mode, the backend should be provided by the caller
|
||||
// to avoid duplicate metrics registration. Only in standalone microservice mode should
|
||||
// this function create its own backend.
|
||||
func NewSearchServer(opts SearchServerOptions) (resource.SearchServer, error) {
|
||||
backend := opts.Backend
|
||||
if backend == nil {
|
||||
eDB, err := dbimpl.ProvideResourceDB(opts.DB, opts.Cfg, opts.Tracer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isHA := isHighAvailabilityEnabled(opts.Cfg.SectionWithEnvOverrides("database"),
|
||||
opts.Cfg.SectionWithEnvOverrides("resource_api"))
|
||||
|
||||
b, err := NewBackend(BackendOptions{
|
||||
DBProvider: eDB,
|
||||
Reg: opts.Reg,
|
||||
IsHA: isHA,
|
||||
LastImportTimeMaxAge: opts.SearchOptions.MaxIndexAge,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize the backend before creating search server
|
||||
if err := b.Init(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize backend: %w", err)
|
||||
}
|
||||
backend = b
|
||||
}
|
||||
|
||||
search, err := resource.NewSearchServer(opts.SearchOptions, backend, opts.AccessClient, nil, opts.IndexMetrics, opts.OwnsIndexFn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create search server: %w", err)
|
||||
}
|
||||
|
||||
if err := search.Init(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize search server: %w", err)
|
||||
}
|
||||
|
||||
return search, nil
|
||||
}
|
||||
|
||||
// NewStorageServer creates a storage-only server without search capabilities.
|
||||
// Use this when you want to run storage and search as separate services.
|
||||
//
|
||||
// Important: When running in monolith mode, the backend should be provided by the caller
|
||||
// to avoid duplicate metrics registration. Only in standalone microservice mode should
|
||||
// this function create its own backend.
|
||||
func NewStorageServer(opts StorageServerOptions) (resource.ResourceServer, error) {
|
||||
apiserverCfg := opts.Cfg.SectionWithEnvOverrides("grafana-apiserver")
|
||||
|
||||
if opts.SecureValues == nil && opts.Cfg != nil && opts.Cfg.SecretsManagement.GrpcClientEnable {
|
||||
@@ -92,7 +154,6 @@ func NewResourceServer(opts ServerOptions) (resource.ResourceServer, error) {
|
||||
|
||||
if opts.Backend != nil {
|
||||
serverOptions.Backend = opts.Backend
|
||||
// TODO: we should probably have a proper interface for diagnostics/lifecycle
|
||||
} else {
|
||||
eDB, err := dbimpl.ProvideResourceDB(opts.DB, opts.Cfg, opts.Tracer)
|
||||
if err != nil {
|
||||
@@ -130,7 +191,6 @@ func NewResourceServer(opts ServerOptions) (resource.ResourceServer, error) {
|
||||
return nil, fmt.Errorf("failed to create resource version manager: %w", err)
|
||||
}
|
||||
|
||||
// TODO add config to decide whether to pass RvManager or not
|
||||
kvBackendOpts.RvManager = rvManager
|
||||
kvBackend, err := resource.NewKVStorageBackend(kvBackendOpts)
|
||||
if err != nil {
|
||||
@@ -148,7 +208,7 @@ func NewResourceServer(opts ServerOptions) (resource.ResourceServer, error) {
|
||||
Reg: opts.Reg,
|
||||
IsHA: isHA,
|
||||
storageMetrics: opts.StorageMetrics,
|
||||
LastImportTimeMaxAge: opts.SearchOptions.MaxIndexAge, // No need to keep last_import_times older than max index age.
|
||||
LastImportTimeMaxAge: opts.Cfg.MaxFileIndexAge,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -159,33 +219,15 @@ func NewResourceServer(opts ServerOptions) (resource.ResourceServer, error) {
|
||||
}
|
||||
}
|
||||
|
||||
serverOptions.Search = opts.SearchOptions
|
||||
serverOptions.IndexMetrics = opts.IndexMetrics
|
||||
// Initialize the backend before creating server
|
||||
if serverOptions.Lifecycle != nil {
|
||||
if err := serverOptions.Lifecycle.Init(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize backend: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
serverOptions.QOSQueue = opts.QOSQueue
|
||||
serverOptions.OwnsIndexFn = opts.OwnsIndexFn
|
||||
serverOptions.OverridesService = opts.OverridesService
|
||||
|
||||
return resource.NewResourceServer(serverOptions)
|
||||
}
|
||||
|
||||
// isHighAvailabilityEnabled determines if high availability mode should
|
||||
// be enabled based on database configuration. High availability is enabled
|
||||
// by default except for SQLite databases.
|
||||
func isHighAvailabilityEnabled(dbCfg, resourceAPICfg *setting.DynamicSection) bool {
|
||||
// If the resource API is using a non-SQLite database, we assume it's in HA mode.
|
||||
resourceDBType := resourceAPICfg.Key("db_type").String()
|
||||
if resourceDBType != "" && resourceDBType != migrator.SQLite {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check in the config if HA is enabled - by default we always assume a HA setup.
|
||||
isHA := dbCfg.Key("high_availability").MustBool(true)
|
||||
|
||||
// SQLite is not possible to run in HA, so we force it to false.
|
||||
databaseType := dbCfg.Key("type").String()
|
||||
if databaseType == migrator.SQLite {
|
||||
isHA = false
|
||||
}
|
||||
|
||||
return isHA
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"go.opentelemetry.io/otel"
|
||||
@@ -36,7 +37,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
"github.com/grafana/grafana/pkg/util/scheduler"
|
||||
)
|
||||
|
||||
@@ -54,38 +55,40 @@ type UnifiedStorageGrpcService interface {
|
||||
type service struct {
|
||||
*services.BasicService
|
||||
|
||||
// Subservices manager
|
||||
subservices *services.Manager
|
||||
subservicesWatcher *services.FailureWatcher
|
||||
hasSubservices bool
|
||||
|
||||
backend resource.StorageBackend
|
||||
cfg *setting.Cfg
|
||||
features featuremgmt.FeatureToggles
|
||||
db infraDB.DB
|
||||
stopCh chan struct{}
|
||||
stoppedCh chan error
|
||||
|
||||
handler grpcserver.Provider
|
||||
|
||||
tracing trace.Tracer
|
||||
|
||||
authenticator func(ctx context.Context) (context.Context, error)
|
||||
|
||||
backend resource.StorageBackend
|
||||
cfg *setting.Cfg
|
||||
features featuremgmt.FeatureToggles
|
||||
stopCh chan struct{}
|
||||
stoppedCh chan error
|
||||
authenticator func(context.Context) (context.Context, error)
|
||||
tracing trace.Tracer
|
||||
db infraDB.DB
|
||||
log log.Logger
|
||||
reg prometheus.Registerer
|
||||
docBuilders resource.DocumentBuilderSupplier
|
||||
storageMetrics *resource.StorageMetrics
|
||||
indexMetrics *resource.BleveIndexMetrics
|
||||
|
||||
docBuilders resource.DocumentBuilderSupplier
|
||||
|
||||
searchRing *ring.Ring
|
||||
|
||||
// Handler for the gRPC server
|
||||
handler grpcserver.Provider
|
||||
|
||||
// Ring lifecycle and sharding support
|
||||
ringLifecycler *ring.BasicLifecycler
|
||||
|
||||
queue QOSEnqueueDequeuer
|
||||
// QoS support
|
||||
queue *scheduler.Queue
|
||||
scheduler *scheduler.Scheduler
|
||||
|
||||
// Subservices management
|
||||
hasSubservices bool
|
||||
subservices *services.Manager
|
||||
subservicesWatcher *services.FailureWatcher
|
||||
}
|
||||
|
||||
// ProvideUnifiedStorageGrpcService provides a combined storage and search service running on the same gRPC server.
|
||||
// This is used when running Grafana as a monolith where both services share the same process.
|
||||
// Each service (storage and search) maintains its own lifecycle but shares the gRPC server.
|
||||
func ProvideUnifiedStorageGrpcService(
|
||||
cfg *setting.Cfg,
|
||||
features featuremgmt.FeatureToggles,
|
||||
@@ -101,7 +104,7 @@ func ProvideUnifiedStorageGrpcService(
|
||||
backend resource.StorageBackend,
|
||||
) (UnifiedStorageGrpcService, error) {
|
||||
var err error
|
||||
tracer := otel.Tracer("unified-storage")
|
||||
tracer := otel.Tracer("unified-storage-combined")
|
||||
|
||||
// FIXME: This is a temporary solution while we are migrating to the new authn interceptor
|
||||
// grpcutils.NewGrpcAuthenticator should be used instead.
|
||||
@@ -178,7 +181,7 @@ func ProvideUnifiedStorageGrpcService(
|
||||
MaxSizePerTenant: cfg.QOSMaxSizePerTenant,
|
||||
Registerer: qosReg,
|
||||
})
|
||||
scheduler, err := scheduler.NewScheduler(queue, &scheduler.Config{
|
||||
sched, err := scheduler.NewScheduler(queue, &scheduler.Config{
|
||||
NumWorkers: cfg.QOSNumberWorker,
|
||||
Logger: log,
|
||||
})
|
||||
@@ -187,7 +190,7 @@ func ProvideUnifiedStorageGrpcService(
|
||||
}
|
||||
|
||||
s.queue = queue
|
||||
s.scheduler = scheduler
|
||||
s.scheduler = sched
|
||||
subservices = append(subservices, s.queue, s.scheduler)
|
||||
}
|
||||
|
||||
@@ -200,6 +203,7 @@ func ProvideUnifiedStorageGrpcService(
|
||||
}
|
||||
|
||||
// This will be used when running as a dskit service
|
||||
// Note: We use StorageServer as the module name for backward compatibility
|
||||
s.BasicService = services.NewBasicService(s.starting, s.running, s.stopping).WithName(modules.StorageServer)
|
||||
|
||||
return s, nil
|
||||
@@ -220,11 +224,6 @@ func (s *service) PrepareDownscale(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
// operation used by the search-servers to check if they own the namespace
|
||||
searchOwnerRead = ring.NewOp([]ring.InstanceState{ring.JOINING, ring.ACTIVE, ring.LEAVING}, nil)
|
||||
)
|
||||
|
||||
func (s *service) OwnsIndex(key resource.NamespacedResource) (bool, error) {
|
||||
if s.searchRing == nil {
|
||||
return true, nil
|
||||
@@ -261,59 +260,108 @@ func (s *service) starting(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
searchOptions, err := search.NewSearchOptions(s.features, s.cfg, s.docBuilders, s.indexMetrics, s.OwnsIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
serverOptions := ServerOptions{
|
||||
Backend: s.backend,
|
||||
DB: s.db,
|
||||
Cfg: s.cfg,
|
||||
Tracer: s.tracing,
|
||||
Reg: s.reg,
|
||||
AccessClient: authzClient,
|
||||
SearchOptions: searchOptions,
|
||||
StorageMetrics: s.storageMetrics,
|
||||
IndexMetrics: s.indexMetrics,
|
||||
Features: s.features,
|
||||
QOSQueue: s.queue,
|
||||
OwnsIndexFn: s.OwnsIndex,
|
||||
}
|
||||
|
||||
// Setup overrides service if enabled
|
||||
var overridesSvc *resource.OverridesService
|
||||
if s.cfg.OverridesFilePath != "" {
|
||||
overridesSvc, err := resource.NewOverridesService(context.Background(), s.log, s.reg, s.tracing, resource.ReloadOptions{
|
||||
overridesSvc, err = resource.NewOverridesService(context.Background(), s.log, s.reg, s.tracing, resource.ReloadOptions{
|
||||
FilePath: s.cfg.OverridesFilePath,
|
||||
ReloadPeriod: s.cfg.OverridesReloadInterval,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serverOptions.OverridesService = overridesSvc
|
||||
}
|
||||
|
||||
server, err := NewResourceServer(serverOptions)
|
||||
// Ensure we have a backend - create one if needed
|
||||
// This is critical: we create the backend ONCE and share it between search and storage servers
|
||||
// to avoid duplicate metrics registration
|
||||
backend := s.backend
|
||||
if backend == nil {
|
||||
eDB, err := dbimpl.ProvideResourceDB(s.db, s.cfg, s.tracing)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create resource DB: %w", err)
|
||||
}
|
||||
|
||||
isHA := isHighAvailabilityEnabled(s.cfg.SectionWithEnvOverrides("database"),
|
||||
s.cfg.SectionWithEnvOverrides("resource_api"))
|
||||
|
||||
b, err := NewBackend(BackendOptions{
|
||||
DBProvider: eDB,
|
||||
Reg: s.reg,
|
||||
IsHA: isHA,
|
||||
storageMetrics: s.storageMetrics,
|
||||
LastImportTimeMaxAge: s.cfg.MaxFileIndexAge,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create backend: %w", err)
|
||||
}
|
||||
|
||||
// Initialize the backend
|
||||
if err := b.Init(context.Background()); err != nil {
|
||||
return fmt.Errorf("failed to initialize backend: %w", err)
|
||||
}
|
||||
backend = b
|
||||
}
|
||||
|
||||
// Create search options for the search server
|
||||
searchOptions, err := search.NewSearchOptions(s.features, s.cfg, s.docBuilders, s.indexMetrics, s.OwnsIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the search server - pass the shared backend
|
||||
searchServer, err := NewSearchServer(SearchServerOptions{
|
||||
Backend: backend, // Use the shared backend
|
||||
DB: s.db,
|
||||
Cfg: s.cfg,
|
||||
Tracer: s.tracing,
|
||||
Reg: s.reg,
|
||||
AccessClient: authzClient,
|
||||
SearchOptions: searchOptions,
|
||||
IndexMetrics: s.indexMetrics,
|
||||
OwnsIndexFn: s.OwnsIndex,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the storage server - pass the shared backend
|
||||
storageServer, err := NewStorageServer(StorageServerOptions{
|
||||
Backend: backend, // Use the shared backend
|
||||
OverridesService: overridesSvc,
|
||||
DB: s.db,
|
||||
Cfg: s.cfg,
|
||||
Tracer: s.tracing,
|
||||
Reg: s.reg,
|
||||
AccessClient: authzClient,
|
||||
StorageMetrics: s.storageMetrics,
|
||||
Features: s.features,
|
||||
QOSQueue: s.queue,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.handler, err = grpcserver.ProvideService(s.cfg, s.features, interceptors.AuthenticatorFunc(s.authenticator), s.tracing, prometheus.DefaultRegisterer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
healthService, err := resource.ProvideHealthService(server)
|
||||
healthService, err := resource.ProvideHealthService(storageServer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srv := s.handler.GetServer()
|
||||
resourcepb.RegisterResourceStoreServer(srv, server)
|
||||
resourcepb.RegisterBulkStoreServer(srv, server)
|
||||
resourcepb.RegisterResourceIndexServer(srv, server)
|
||||
resourcepb.RegisterManagedObjectIndexServer(srv, server)
|
||||
resourcepb.RegisterBlobStoreServer(srv, server)
|
||||
resourcepb.RegisterDiagnosticsServer(srv, server)
|
||||
resourcepb.RegisterQuotasServer(srv, server)
|
||||
// Register storage services
|
||||
resourcepb.RegisterResourceStoreServer(srv, storageServer)
|
||||
resourcepb.RegisterBulkStoreServer(srv, storageServer)
|
||||
resourcepb.RegisterBlobStoreServer(srv, storageServer)
|
||||
resourcepb.RegisterDiagnosticsServer(srv, storageServer)
|
||||
resourcepb.RegisterQuotasServer(srv, storageServer)
|
||||
// Register search services
|
||||
resourcepb.RegisterResourceIndexServer(srv, searchServer)
|
||||
resourcepb.RegisterManagedObjectIndexServer(srv, searchServer)
|
||||
grpc_health_v1.RegisterHealthServer(srv, healthService)
|
||||
|
||||
// register reflection service
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"google.golang.org/grpc/health/grpc_health_v1"
|
||||
|
||||
"github.com/grafana/dskit/services"
|
||||
|
||||
infraDB "github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/modules"
|
||||
"github.com/grafana/grafana/pkg/services/authz"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/grpcserver"
|
||||
"github.com/grafana/grafana/pkg/services/grpcserver/interceptors"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||
"github.com/grafana/grafana/pkg/util/scheduler"
|
||||
)
|
||||
|
||||
var _ UnifiedStorageGrpcService = (*storageService)(nil)
|
||||
|
||||
type storageService struct {
|
||||
*services.BasicService
|
||||
|
||||
backend resource.StorageBackend
|
||||
cfg *setting.Cfg
|
||||
features featuremgmt.FeatureToggles
|
||||
db infraDB.DB
|
||||
stopCh chan struct{}
|
||||
stoppedCh chan error
|
||||
|
||||
handler grpcserver.Provider
|
||||
|
||||
tracing trace.Tracer
|
||||
|
||||
authenticator func(ctx context.Context) (context.Context, error)
|
||||
|
||||
log log.Logger
|
||||
reg prometheus.Registerer
|
||||
storageMetrics *resource.StorageMetrics
|
||||
|
||||
queue QOSEnqueueDequeuer
|
||||
scheduler *scheduler.Scheduler
|
||||
|
||||
// Subservices manager
|
||||
subservices *services.Manager
|
||||
subservicesWatcher *services.FailureWatcher
|
||||
hasSubservices bool
|
||||
}
|
||||
|
||||
func ProvideStorageService(
|
||||
cfg *setting.Cfg,
|
||||
features featuremgmt.FeatureToggles,
|
||||
db infraDB.DB,
|
||||
log log.Logger,
|
||||
reg prometheus.Registerer,
|
||||
storageMetrics *resource.StorageMetrics,
|
||||
backend resource.StorageBackend,
|
||||
) (UnifiedStorageGrpcService, error) {
|
||||
var err error
|
||||
tracer := otel.Tracer("unified-storage-server")
|
||||
|
||||
authn := NewAuthenticatorWithFallback(cfg, reg, tracer, func(ctx context.Context) (context.Context, error) {
|
||||
auth := grpc.Authenticator{Tracer: tracer}
|
||||
return auth.Authenticate(ctx)
|
||||
})
|
||||
|
||||
s := &storageService{
|
||||
backend: backend,
|
||||
cfg: cfg,
|
||||
features: features,
|
||||
stopCh: make(chan struct{}),
|
||||
stoppedCh: make(chan error, 1),
|
||||
authenticator: authn,
|
||||
tracing: tracer,
|
||||
db: db,
|
||||
log: log,
|
||||
reg: reg,
|
||||
storageMetrics: storageMetrics,
|
||||
subservicesWatcher: services.NewFailureWatcher(),
|
||||
}
|
||||
|
||||
subservices := []services.Service{}
|
||||
|
||||
// Setup QOS if enabled
|
||||
if cfg.QOSEnabled {
|
||||
qosReg := prometheus.WrapRegistererWithPrefix("resource_server_qos_", reg)
|
||||
queue := scheduler.NewQueue(&scheduler.QueueOptions{
|
||||
MaxSizePerTenant: cfg.QOSMaxSizePerTenant,
|
||||
Registerer: qosReg,
|
||||
})
|
||||
sched, err := scheduler.NewScheduler(queue, &scheduler.Config{
|
||||
NumWorkers: cfg.QOSNumberWorker,
|
||||
Logger: log,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create qos scheduler: %s", err)
|
||||
}
|
||||
|
||||
s.queue = queue
|
||||
s.scheduler = sched
|
||||
subservices = append(subservices, s.queue, s.scheduler)
|
||||
}
|
||||
|
||||
if len(subservices) > 0 {
|
||||
s.hasSubservices = true
|
||||
s.subservices, err = services.NewManager(subservices...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create subservices manager: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// This will be used when running as a dskit service
|
||||
s.BasicService = services.NewBasicService(s.starting, s.running, s.stopping).WithName(modules.StorageServer)
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *storageService) starting(ctx context.Context) error {
|
||||
if s.hasSubservices {
|
||||
s.subservicesWatcher.WatchManager(s.subservices)
|
||||
if err := services.StartManagerAndAwaitHealthy(ctx, s.subservices); err != nil {
|
||||
return fmt.Errorf("failed to start subservices: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
authzClient, err := authz.ProvideStandaloneAuthZClient(s.cfg, s.features, s.tracing, s.reg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Setup overrides service if enabled
|
||||
var overridesSvc *resource.OverridesService
|
||||
if s.cfg.OverridesFilePath != "" {
|
||||
overridesSvc, err = resource.NewOverridesService(context.Background(), s.log, s.reg, s.tracing, resource.ReloadOptions{
|
||||
FilePath: s.cfg.OverridesFilePath,
|
||||
ReloadPeriod: s.cfg.OverridesReloadInterval,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Create the storage server
|
||||
storageServer, err := NewStorageServer(StorageServerOptions{
|
||||
Backend: s.backend,
|
||||
OverridesService: overridesSvc,
|
||||
DB: s.db,
|
||||
Cfg: s.cfg,
|
||||
Tracer: s.tracing,
|
||||
Reg: s.reg,
|
||||
AccessClient: authzClient,
|
||||
StorageMetrics: s.storageMetrics,
|
||||
Features: s.features,
|
||||
QOSQueue: s.queue,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.handler, err = grpcserver.ProvideService(s.cfg, s.features, interceptors.AuthenticatorFunc(s.authenticator), s.tracing, prometheus.DefaultRegisterer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
healthService, err := resource.ProvideHealthService(storageServer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srv := s.handler.GetServer()
|
||||
// Register storage services
|
||||
resourcepb.RegisterResourceStoreServer(srv, storageServer)
|
||||
resourcepb.RegisterBulkStoreServer(srv, storageServer)
|
||||
resourcepb.RegisterBlobStoreServer(srv, storageServer)
|
||||
resourcepb.RegisterDiagnosticsServer(srv, storageServer)
|
||||
resourcepb.RegisterQuotasServer(srv, storageServer)
|
||||
grpc_health_v1.RegisterHealthServer(srv, healthService)
|
||||
|
||||
// register reflection service
|
||||
_, err = grpcserver.ProvideReflectionService(s.cfg, s.handler)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// start the gRPC server
|
||||
go func() {
|
||||
err := s.handler.Run(ctx)
|
||||
if err != nil {
|
||||
s.stoppedCh <- err
|
||||
} else {
|
||||
s.stoppedCh <- nil
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAddress returns the address of the gRPC server.
|
||||
func (s *storageService) GetAddress() string {
|
||||
return s.handler.GetAddress()
|
||||
}
|
||||
|
||||
func (s *storageService) running(ctx context.Context) error {
|
||||
select {
|
||||
case err := <-s.stoppedCh:
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
case err := <-s.subservicesWatcher.Chan():
|
||||
return fmt.Errorf("subservice failure: %w", err)
|
||||
case <-ctx.Done():
|
||||
close(s.stopCh)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *storageService) stopping(_ error) error {
|
||||
if s.hasSubservices {
|
||||
err := services.StopManagerAndAwaitStopped(context.Background(), s.subservices)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stop subservices: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -32,6 +32,7 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
nsPrefix := "test-ns"
|
||||
|
||||
var server resource.ResourceServer
|
||||
var searchServer resource.SearchServer
|
||||
|
||||
t.Run("Create initial resources in storage", func(t *testing.T) {
|
||||
initialResources := []struct {
|
||||
@@ -96,25 +97,34 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
})
|
||||
|
||||
t.Run("Create a resource server with both backends", func(t *testing.T) {
|
||||
// Create a resource server with both backends
|
||||
// Create search server first
|
||||
var err error
|
||||
server, err = resource.NewResourceServer(resource.ResourceServerOptions{
|
||||
Backend: backend,
|
||||
Search: resource.SearchOptions{
|
||||
Backend: searchBackend,
|
||||
Resources: &resource.TestDocumentBuilderSupplier{
|
||||
GroupsResources: map[string]string{
|
||||
"test.grafana.app": "testresources",
|
||||
},
|
||||
searchOpts := resource.SearchOptions{
|
||||
Backend: searchBackend,
|
||||
Resources: &resource.TestDocumentBuilderSupplier{
|
||||
GroupsResources: map[string]string{
|
||||
"test.grafana.app": "testresources",
|
||||
},
|
||||
},
|
||||
}
|
||||
searchServer, err = resource.NewSearchServer(searchOpts, backend, nil, nil, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, searchServer)
|
||||
|
||||
// Initialize the search server
|
||||
err = searchServer.Init(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a resource server separately from the search server
|
||||
server, err = resource.NewResourceServer(resource.ResourceServerOptions{
|
||||
Backend: backend,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("Search for initial resources", func(t *testing.T) {
|
||||
// Test 1: Search for initial resources
|
||||
searchResp, err := server.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
searchResp, err := searchServer.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
Options: &resourcepb.ListOptions{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Group: "test.grafana.app",
|
||||
@@ -194,7 +204,7 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
})
|
||||
|
||||
t.Run("Search for documents", func(t *testing.T) {
|
||||
searchResp, err := server.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
searchResp, err := searchServer.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
Options: &resourcepb.ListOptions{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Group: "test.grafana.app",
|
||||
@@ -212,7 +222,7 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
})
|
||||
|
||||
t.Run("Search with tags", func(t *testing.T) {
|
||||
searchResp, err := server.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
searchResp, err := searchServer.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
Options: &resourcepb.ListOptions{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Group: "test.grafana.app",
|
||||
@@ -231,7 +241,7 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
require.Equal(t, int64(0), searchResp.TotalHits)
|
||||
|
||||
// this is the correct way of searching by tag
|
||||
searchResp, err = server.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
searchResp, err = searchServer.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
Options: &resourcepb.ListOptions{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Group: "test.grafana.app",
|
||||
@@ -253,7 +263,7 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
})
|
||||
|
||||
t.Run("Search with specific tag", func(t *testing.T) {
|
||||
searchResp, err := server.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
searchResp, err := searchServer.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
Options: &resourcepb.ListOptions{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Group: "test.grafana.app",
|
||||
@@ -272,7 +282,7 @@ func RunTestSearchAndStorage(t *testing.T, ctx context.Context, backend resource
|
||||
require.Equal(t, int64(0), searchResp.TotalHits)
|
||||
|
||||
// this is the correct way of searching by tag
|
||||
searchResp, err = server.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
searchResp, err = searchServer.Search(ctx, &resourcepb.ResourceSearchRequest{
|
||||
Options: &resourcepb.ListOptions{
|
||||
Key: &resourcepb.ResourceKey{
|
||||
Group: "test.grafana.app",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user