public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into ds-apiserver-with-configs

Browse Source
This commit is contained in:
Ryan McKinley
2025-07-02 09:56:20 -07:00
parent a44fdee0ef d9a8253640
commit c0f1a6423c
398 changed files with 7285 additions and 5179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.betterer.results
+210 -40
View File
@@ -489,6 +489,9 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "6"],
[0, 0, 0, "Unexpected any. Specify a different type.", "7"]
],
"packages/grafana-prometheus/src/test/mocks/datasource.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"packages/grafana-prometheus/src/types.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -958,6 +961,9 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not re-export imported variable (\`profiler\`)", "6"],
[0, 0, 0, "Do not re-export imported variable (\`updateLegendValues\`)", "7"]
],
"public/app/core/navigation/mocks/routeProps.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/core/navigation/types.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]
],
@@ -1566,6 +1572,9 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"],
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "1"]
],
"public/app/features/correlations/mocks/useCorrelations.mocks.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/features/correlations/types.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"]
@@ -2223,10 +2232,6 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"],
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "1"]
],
"public/app/features/datasources/__mocks__/index.ts:5381": [
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"],
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "1"]
],
"public/app/features/datasources/components/DataSourceTypeCard.tsx:5381": [
[0, 0, 0, "Add noMargin prop to Card components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "0"],
[0, 0, 0, "Use data-testid for E2E selectors instead of aria-label", "1"]
@@ -2597,11 +2602,6 @@ exports[`better eslint`] = {
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "1"],
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "2"]
],
"public/app/features/plugins/admin/__mocks__/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./localPlugin.mock\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./remotePlugin.mock\`)", "1"],
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "2"]
],
"public/app/features/plugins/admin/components/Badges/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./PluginDeprecatedBadge\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./PluginDisabledBadge\`)", "1"],
@@ -2622,6 +2622,16 @@ exports[`better eslint`] = {
"public/app/features/plugins/admin/components/PluginDetailsPage.tsx:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/features/plugins/admin/mocks/catalogPlugin.mock.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/features/plugins/admin/mocks/localPlugin.mock.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/features/plugins/admin/mocks/remotePlugin.mock.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/features/plugins/admin/pages/Browse.tsx:5381": [
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "0"],
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "1"],
@@ -2972,6 +2982,12 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not use any type assertions.", "2"],
[0, 0, 0, "Do not use any type assertions.", "3"]
],
"public/app/features/users/mocks/userMocks.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"],
[0, 0, 0, "Do not use any type assertions.", "2"],
[0, 0, 0, "Do not use any type assertions.", "3"]
],
"public/app/features/variables/adapters.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -3196,6 +3212,21 @@ exports[`better eslint`] = {
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "5"],
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "6"]
],
"public/app/plugins/datasource/azuremonitor/mocks/datasource.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/azuremonitor/mocks/panelData.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"],
[0, 0, 0, "Do not use any type assertions.", "2"]
],
"public/app/plugins/datasource/azuremonitor/mocks/utils.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/plugins/datasource/azuremonitor/mocks/variables.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/azuremonitor/types/index.ts:5381": [
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"],
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "1"],
@@ -3260,6 +3291,14 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/plugins/datasource/cloud-monitoring/mocks/cloudMonitoringDatasource.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"],
[0, 0, 0, "Do not use any type assertions.", "2"]
],
"public/app/plugins/datasource/cloud-monitoring/mocks/cloudMonitoringInstanceSettings.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloud-monitoring/types/query.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`../dataquery.gen\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`../dataquery.gen\`)", "1"],
@@ -3271,35 +3310,6 @@ exports[`better eslint`] = {
"public/app/plugins/datasource/cloud-monitoring/webpack.config.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`config\`)", "0"]
],
"public/app/plugins/datasource/cloudwatch/__mocks__/cloudwatch-logs-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./commentOnlyQuery\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./empty\`)", "1"],
[0, 0, 0, "Do not re-export imported variable (\`./filterQuery\`)", "2"],
[0, 0, 0, "Do not re-export imported variable (\`./multiLineFullQuery\`)", "3"],
[0, 0, 0, "Do not re-export imported variable (\`./newCommandQuery\`)", "4"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineFullQuery\`)", "5"],
[0, 0, 0, "Do not re-export imported variable (\`./sortQuery\`)", "6"],
[0, 0, 0, "Do not re-export imported variable (\`./whitespaceQuery\`)", "7"]
],
"public/app/plugins/datasource/cloudwatch/__mocks__/cloudwatch-sql-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./multiLineFullQuery\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./multiLineIncompleteQueryWithoutNamespace\`)", "1"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineEmptyQuery\`)", "2"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineFullQuery\`)", "3"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineTwoQueries\`)", "4"]
],
"public/app/plugins/datasource/cloudwatch/__mocks__/dynamic-label-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./afterLabelValue\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./insideLabelValue\`)", "1"]
],
"public/app/plugins/datasource/cloudwatch/__mocks__/metric-math-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./afterFunctionQuery\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./secondArgAfterSearchQuery\`)", "1"],
[0, 0, 0, "Do not re-export imported variable (\`./secondArgQuery\`)", "2"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineEmptyQuery\`)", "3"],
[0, 0, 0, "Do not re-export imported variable (\`./thirdArgAfterSearchQuery\`)", "4"],
[0, 0, 0, "Do not re-export imported variable (\`./withinStringQuery\`)", "5"]
],
"public/app/plugins/datasource/cloudwatch/components/ConfigEditor/ConfigEditor.tsx:5381": [
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "0"],
[0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "1"],
@@ -3337,6 +3347,153 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
[0, 0, 0, "Unexpected any. Specify a different type.", "2"]
],
"public/app/plugins/datasource/cloudwatch/mocks/CloudWatchVariables.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/commentOnlyQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/multiLineFullQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/multiLineFullQueryWithCaseClause.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/partialQueryWithFunction.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/partialQueryWithSubquery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/singleLineFullQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-sql-test-data/whitespaceQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/commentOnlyQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/filterQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./commentOnlyQuery\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./empty\`)", "1"],
[0, 0, 0, "Do not re-export imported variable (\`./filterQuery\`)", "2"],
[0, 0, 0, "Do not re-export imported variable (\`./multiLineFullQuery\`)", "3"],
[0, 0, 0, "Do not re-export imported variable (\`./newCommandQuery\`)", "4"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineFullQuery\`)", "5"],
[0, 0, 0, "Do not re-export imported variable (\`./sortQuery\`)", "6"],
[0, 0, 0, "Do not re-export imported variable (\`./whitespaceQuery\`)", "7"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/multiLineFullQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/newCommandQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/singleLineFullQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/sortQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-logs-test-data/whitespaceQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-ppl-test-data/multilineQueries.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-ppl-test-data/newCommandQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-ppl-test-data/singleLineQueries.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"],
[0, 0, 0, "Do not use any type assertions.", "2"],
[0, 0, 0, "Do not use any type assertions.", "3"],
[0, 0, 0, "Do not use any type assertions.", "4"],
[0, 0, 0, "Do not use any type assertions.", "5"],
[0, 0, 0, "Do not use any type assertions.", "6"],
[0, 0, 0, "Do not use any type assertions.", "7"],
[0, 0, 0, "Do not use any type assertions.", "8"],
[0, 0, 0, "Do not use any type assertions.", "9"],
[0, 0, 0, "Do not use any type assertions.", "10"],
[0, 0, 0, "Do not use any type assertions.", "11"],
[0, 0, 0, "Do not use any type assertions.", "12"],
[0, 0, 0, "Do not use any type assertions.", "13"],
[0, 0, 0, "Do not use any type assertions.", "14"],
[0, 0, 0, "Do not use any type assertions.", "15"],
[0, 0, 0, "Do not use any type assertions.", "16"],
[0, 0, 0, "Do not use any type assertions.", "17"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-sql-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./multiLineFullQuery\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./multiLineIncompleteQueryWithoutNamespace\`)", "1"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineEmptyQuery\`)", "2"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineFullQuery\`)", "3"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineTwoQueries\`)", "4"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-sql-test-data/multiLineFullQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-sql-test-data/multiLineIncompleteQueryWithoutNamespace.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-sql-test-data/singleLineEmptyQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-sql-test-data/singleLineFullQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/cloudwatch-sql-test-data/singleLineTwoQueries.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/dashboardOnLoadedEvent.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/dynamic-label-test-data/afterLabelValue.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/dynamic-label-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./afterLabelValue\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./insideLabelValue\`)", "1"]
],
"public/app/plugins/datasource/cloudwatch/mocks/dynamic-label-test-data/insideLabelValue.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/afterFunctionQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/index.ts:5381": [
[0, 0, 0, "Do not re-export imported variable (\`./afterFunctionQuery\`)", "0"],
[0, 0, 0, "Do not re-export imported variable (\`./secondArgAfterSearchQuery\`)", "1"],
[0, 0, 0, "Do not re-export imported variable (\`./secondArgQuery\`)", "2"],
[0, 0, 0, "Do not re-export imported variable (\`./singleLineEmptyQuery\`)", "3"],
[0, 0, 0, "Do not re-export imported variable (\`./thirdArgAfterSearchQuery\`)", "4"],
[0, 0, 0, "Do not re-export imported variable (\`./withinStringQuery\`)", "5"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/secondArgAfterSearchQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/secondArgQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/singleLineEmptyQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/thirdArgAfterSearchQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/metric-math-test-data/withinStringQuery.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/cloudwatch/mocks/monarch/Monaco.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/plugins/datasource/cloudwatch/types.ts:5381": [
[0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -3409,6 +3566,9 @@ exports[`better eslint`] = {
"public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/state/reducer.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/elasticsearch/configuration/mocks/configOptions.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/elasticsearch/datasource.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -3539,8 +3699,7 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "15"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"],
[0, 0, 0, "Unexpected any. Specify a different type.", "17"],
[0, 0, 0, "Unexpected any. Specify a different type.", "18"],
[0, 0, 0, "Unexpected any. Specify a different type.", "19"]
[0, 0, 0, "Unexpected any. Specify a different type.", "18"]
],
"public/app/plugins/datasource/graphite/gfunc.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
@@ -3628,6 +3787,10 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "7"],
[0, 0, 0, "Unexpected any. Specify a different type.", "8"]
],
"public/app/plugins/datasource/influxdb/mocks/datasource.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/plugins/datasource/influxdb/query_part.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
@@ -3695,6 +3858,13 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"]
],
"public/app/plugins/datasource/loki/mocks/datasource.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
],
"public/app/plugins/datasource/loki/mocks/metadataRequest.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/loki/querybuilder/components/LokiQueryBuilder.tsx:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
.github/CODEOWNERS
-1
View File
@@ -180,7 +180,6 @@
/pkg/tests/apis/query @grafana/grafana-datasources-core-services
/pkg/tests/apis/alerting @grafana/grafana-app-platform-squad @grafana/alerting-backend
/pkg/tests/api/correlations/ @grafana/datapro
/pkg/tests/apis/secret/ @grafana/grafana-operator-experience-squad
/pkg/tsdb/grafanads/ @grafana/grafana-backend-group
/pkg/tsdb/opentsdb/ @grafana/partner-datasources
/pkg/util/ @grafana/grafana-backend-group
devenv/dev-dashboards/panel-timeseries/timeseries-nulls.json
+2 -3
View File
@@ -170,8 +170,7 @@
"type": "grafana-testdata-datasource",
"uid": "PD8C576611E62080A"
},
"rawFrameContent": "[\n {\n \"schema\": {\n \"refId\": \"A\",\n \"name\": \"A\",\n \"fields\": [\n {\n \"name\": \"Time\",\n \"type\": \"time\",\n \"typeInfo\": {\n \"frame\": \"time.Time\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Value\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Min\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Max\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Info\",\n \"type\": \"string\",\n \"typeInfo\": {\n \"frame\": \"string\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Status\",\n \"type\": \"string\",\n \"typeInfo\": {\n \"frame\": \"string\",\n \"nullable\": true\n },\n \"config\": {}\n }\n ]\n },\n \"data\": {\n \"values\": [\n [\n \"2023-06-14T08:16:41Z\",\n \"2023-06-14T08:16:50Z\",\n \"2023-06-14T08:16:55Z\",\n \"2023-06-14T08:17:15Z\",\n \"2023-06-14T08:17:46Z\",\n \"2023-06-14T08:17:59Z\",\n \"2023-06-14T08:18:20Z\",\n \"2023-06-14T08:18:25Z\"\n ],\n [\n 97.56700088585262,\n 97.51875596242603,\n 97.88367836664669,\n 97.64813860191066,\n 97.46344237059202,\n 97.06932713353162,\n 97.11185691118344,\n 97.4315327939052\n ],\n [\n 96.17381933561288,\n 96.86449303671965,\n 95.5851820937877,\n 96.94658694041154,\n 96.7433590164638,\n 96.74913605945137,\n 95.61938577725707,\n 96.24186355022567\n ],\n [\n 99.69898997339244,\n 98.68620205532211,\n 99.5800005905314,\n 98.2998754185445,\n 98.02873998081307,\n 99.01798295660049,\n 97.95260164072228,\n 98.24811606848421\n ],\n [\n \"down\",\n \"down\",\n \"up\",\n \"down\",\n \"down\",\n \"down\",\n \"up\",\n \"up\"\n ],\n [\n \"ON\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"STANDBY\",\n \"OFF\"\n ]\n ]\n }\n }\n]",
"refId": "A",
"rawFrameContent": "[\n {\n \"schema\": {\n \"refId\": \"A\",\n \"name\": \"A\",\n \"fields\": [\n {\n \"name\": \"Time\",\n \"type\": \"time\",\n \"typeInfo\": {\n \"frame\": \"time.Time\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Value\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Min\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Max\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Status\",\n \"type\": \"string\",\n \"typeInfo\": {\n \"frame\": \"string\",\n \"nullable\": true\n },\n \"config\": {}\n }\n ]\n },\n \"data\": {\n \"values\": [\n [\n 1686730601000,\n 1686730610000,\n 1686730615000,\n 1686730635000,\n 1686730666000,\n 1686730679000,\n 1686730700000,\n 1686730705000\n ],\n [\n 97.56700088585262,\n 97.51875596242603,\n 97.88367836664669,\n 97.64813860191066,\n 97.46344237059202,\n 97.06932713353162,\n 97.11185691118344,\n 97.4315327939052\n ],\n [\n 96.17381933561288,\n 96.86449303671965,\n 95.5851820937877,\n 96.94658694041154,\n 96.7433590164638,\n 96.74913605945137,\n 95.61938577725707,\n 96.24186355022567\n ],\n [\n 99.69898997339244,\n 98.68620205532211,\n 99.5800005905314,\n 98.2998754185445,\n 98.02873998081307,\n 99.01798295660049,\n 97.95260164072228,\n 98.24811606848421\n ],\n [\n \"ON\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"STANDBY\",\n \"OFF\"\n ]\n ]\n }\n }\n]", "refId": "A",
"scenarioId": "raw_frame"
}
],
@@ -292,7 +291,7 @@
"type": "grafana-testdata-datasource",
"uid": "PD8C576611E62080A"
},
"rawFrameContent": "[\n {\n \"schema\": {\n \"refId\": \"A\",\n \"name\": \"A\",\n \"fields\": [\n {\n \"name\": \"Time\",\n \"type\": \"time\",\n \"typeInfo\": {\n \"frame\": \"time.Time\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Value\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Min\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Max\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Status\",\n \"type\": \"string\",\n \"typeInfo\": {\n \"frame\": \"string\",\n \"nullable\": true\n },\n \"config\": {}\n }\n ]\n },\n \"data\": {\n \"values\": [\n [\n \"2023-06-14T08:16:41Z\",\n \"2023-06-14T08:16:50Z\",\n \"2023-06-14T08:16:55Z\",\n \"2023-06-14T08:17:15Z\",\n \"2023-06-14T08:17:46Z\",\n \"2023-06-14T08:17:59Z\",\n \"2023-06-14T08:18:20Z\",\n \"2023-06-14T08:18:25Z\"\n ],\n [\n 97.56700088585262,\n 97.51875596242603,\n 97.88367836664669,\n 97.64813860191066,\n 97.46344237059202,\n 97.06932713353162,\n 97.11185691118344,\n 97.4315327939052\n ],\n [\n 96.17381933561288,\n 96.86449303671965,\n 95.5851820937877,\n 96.94658694041154,\n 96.7433590164638,\n 96.74913605945137,\n 95.61938577725707,\n 96.24186355022567\n ],\n [\n 99.69898997339244,\n 98.68620205532211,\n 99.5800005905314,\n 98.2998754185445,\n 98.02873998081307,\n 99.01798295660049,\n 97.95260164072228,\n 98.24811606848421\n ],\n [\n \"ON\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"STANDBY\",\n \"OFF\"\n ]\n ]\n }\n }\n]",
"rawFrameContent": "[\n {\n \"schema\": {\n \"refId\": \"A\",\n \"name\": \"A\",\n \"fields\": [\n {\n \"name\": \"Time\",\n \"type\": \"time\",\n \"typeInfo\": {\n \"frame\": \"time.Time\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Value\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Min\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Max\",\n \"type\": \"number\",\n \"typeInfo\": {\n \"frame\": \"float64\",\n \"nullable\": true\n },\n \"config\": {}\n },\n {\n \"name\": \"Status\",\n \"type\": \"string\",\n \"typeInfo\": {\n \"frame\": \"string\",\n \"nullable\": true\n },\n \"config\": {}\n }\n ]\n },\n \"data\": {\n \"values\": [\n [\n 1686730601000,\n 1686730610000,\n 1686730615000,\n 1686730635000,\n 1686730666000,\n 1686730679000,\n 1686730700000,\n 1686730705000\n ],\n [\n 97.56700088585262,\n 97.51875596242603,\n 97.88367836664669,\n 97.64813860191066,\n 97.46344237059202,\n 97.06932713353162,\n 97.11185691118344,\n 97.4315327939052\n ],\n [\n 96.17381933561288,\n 96.86449303671965,\n 95.5851820937877,\n 96.94658694041154,\n 96.7433590164638,\n 96.74913605945137,\n 95.61938577725707,\n 96.24186355022567\n ],\n [\n 99.69898997339244,\n 98.68620205532211,\n 99.5800005905314,\n 98.2998754185445,\n 98.02873998081307,\n 99.01798295660049,\n 97.95260164072228,\n 98.24811606848421\n ],\n [\n \"ON\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"OFF\",\n \"ON\",\n \"STANDBY\",\n \"OFF\"\n ]\n ]\n }\n }\n]",
"refId": "A",
"scenarioId": "raw_frame"
}
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
+1 -1
View File
@@ -53,7 +53,6 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
| `ssoSettingsApi` | Enables the SSO settings API and the OAuth configuration UIs in Grafana | Yes |
| `logsInfiniteScrolling` | Enables infinite scrolling for the Logs panel in Explore and Dashboards | Yes |
| `logRowsPopoverMenu` | Enable filtering menu displayed when text of a log line is selected | Yes |
| `lokiQueryHints` | Enables query hints for Loki | Yes |
| `alertingQueryOptimization` | Optimizes eligible queries in order to reduce load on datasources | |
| `onPremToCloudMigrations` | Enable the Grafana Migration Assistant, which helps you easily migrate various on-prem resources to your Grafana Cloud stack. | Yes |
| `groupToNestedTableTransformation` | Enables the group to nested table transformation | Yes |
@@ -103,6 +102,7 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
| `cloudWatchBatchQueries` | Runs CloudWatch metrics queries as separate batches |
| `pdfTables` | Enables generating table data as PDF in reporting |
| `canvasPanelPanZoom` | Allow pan and zoom in canvas panel |
| `regressionTransformation` | Enables regression analysis transformation |
| `alertingSaveStateCompressed` | Enables the compressed protobuf-based alert state storage |
| `tableNextGen` | Allows access to the new react-data-grid based table component. |
| `enableSCIM` | Enables SCIM support for user and group management |
docs/sources/setup-grafana/configure-security/configure-team-sync.md
+2 -2
View File
@@ -33,8 +33,8 @@ This mechanism allows Grafana to remove an existing synchronized user from a tea
- [GitHub OAuth](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/github/#configure-team-synchronization)
- [GitLab OAuth](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/gitlab/#configure-team-synchronization)
- [Google OAuth](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/google/#configure-team-synchronization)
- [LDAP](https://grafana.com/docs/grafana/<GRAFANA_VERSION/setup-grafana/configure-security/configure-authentication/enhanced-ldap/)
- [Okta](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/saml/)
- [LDAP](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/enhanced-ldap/)
- [Okta](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/okta/#configure-team-synchronization-enterprise-only)
- [SAML](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/setup-grafana/configure-security/configure-authentication/saml/)
## Synchronize a Grafana team with an external group
package.json
+2 -2
View File
@@ -289,8 +289,8 @@
"@grafana/plugin-ui": "0.10.7",
"@grafana/prometheus": "workspace:*",
"@grafana/runtime": "workspace:*",
"@grafana/scenes": "^6.24.0",
"@grafana/scenes-react": "^6.24.0",
"@grafana/scenes": "^6.26.1",
"@grafana/scenes-react": "^6.26.1",
"@grafana/schema": "workspace:*",
"@grafana/sql": "workspace:*",
"@grafana/ui": "workspace:*",
packages/grafana-data/src/types/featureToggles.gen.ts
+2 -3
View File
@@ -420,10 +420,9 @@ export interface FeatureToggles {
*/
tableSharedCrosshair?: boolean;
/**
* Enables query hints for Loki
* @default true
* Enables regression analysis transformation
*/
lokiQueryHints?: boolean;
regressionTransformation?: boolean;
/**
* Use the kubernetes API for feature toggle management in the frontend
*/
packages/grafana-prometheus/src/components/metrics-browser/MetricsBrowserContext.test.tsx
+1 -1
View File
@@ -7,7 +7,7 @@ import { TimeRange } from '@grafana/data';
import { DEFAULT_SERIES_LIMIT, LAST_USED_LABELS_KEY, METRIC_LABEL } from '../../constants';
import { PrometheusDatasource } from '../../datasource';
import { PrometheusLanguageProviderInterface } from '../../language_provider';
import { getMockTimeRange } from '../../test/__mocks__/datasource';
import { getMockTimeRange } from '../../test/mocks/datasource';
import { MetricsBrowserProvider, useMetricsBrowser } from './MetricsBrowserContext';
packages/grafana-prometheus/src/components/metrics-browser/useMetricsLabelsValues.test.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import { TimeRange } from '@grafana/data';
import { DEFAULT_SERIES_LIMIT, EMPTY_SELECTOR, LAST_USED_LABELS_KEY, METRIC_LABEL } from '../../constants';
import { PrometheusDatasource } from '../../datasource';
import { PrometheusLanguageProvider, PrometheusLanguageProviderInterface } from '../../language_provider';
import { getMockTimeRange } from '../../test/__mocks__/datasource';
import { getMockTimeRange } from '../../test/mocks/datasource';
import * as selectorBuilderModule from './selectorBuilder';
import { useMetricsLabelsValues } from './useMetricsLabelsValues';
packages/grafana-prometheus/src/components/monaco-query-field/monaco-completion-provider/completions.test.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { config } from '@grafana/runtime';
import { SUGGESTIONS_LIMIT } from '../../../constants';
import { FUNCTIONS } from '../../../promql';
import { getMockTimeRange } from '../../../test/__mocks__/datasource';
import { getMockTimeRange } from '../../../test/mocks/datasource';
import { filterMetricNames, getCompletions } from './completions';
import { DataProvider, type DataProviderParams } from './data_provider';
packages/grafana-prometheus/src/configuration/AlertingSettingsOverhaul.test.tsx
+1 -1
View File
@@ -3,7 +3,7 @@ import { render } from '@testing-library/react';
import { selectors } from '@grafana/e2e-selectors';
import { config } from '@grafana/runtime';
import { createDefaultConfigOptions } from '../test/__mocks__/datasource';
import { createDefaultConfigOptions } from '../test/mocks/datasource';
import { AlertingSettingsOverhaul } from './AlertingSettingsOverhaul';
packages/grafana-prometheus/src/configuration/PromSettings.test.tsx
+1 -1
View File
@@ -7,7 +7,7 @@ import { selectors } from '@grafana/e2e-selectors';
import { config } from '@grafana/runtime';
import { countError } from '../constants';
import { createDefaultConfigOptions } from '../test/__mocks__/datasource';
import { createDefaultConfigOptions } from '../test/mocks/datasource';
import { getValueFromEventItem, PromSettings } from './PromSettings';
packages/grafana-prometheus/src/datasource.test.ts
+1 -1
View File
@@ -24,7 +24,7 @@ import {
createDefaultPromResponse,
fetchMockCalledWith,
getMockTimeRange,
} from './test/__mocks__/datasource';
} from './test/mocks/datasource';
import {
PromApplication,
PrometheusCacheLevel,
packages/grafana-prometheus/src/querybuilder/components/MetricCombobox.test.tsx
+1 -1
View File
@@ -8,7 +8,7 @@ import { DataSourceInstanceSettings } from '@grafana/data';
import { PrometheusDatasource } from '../../datasource';
import { PrometheusLanguageProviderInterface } from '../../language_provider';
import { EmptyLanguageProviderMock } from '../../language_provider.mock';
import { getMockTimeRange } from '../../test/__mocks__/datasource';
import { getMockTimeRange } from '../../test/mocks/datasource';
import { PromOptions } from '../../types';
import { MetricCombobox, MetricComboboxProps } from './MetricCombobox';
packages/grafana-prometheus/src/querybuilder/components/metrics-modal/MetricsModal.test.tsx
+1 -1
View File
@@ -7,7 +7,7 @@ import { DataSourceInstanceSettings, DataSourcePluginMeta } from '@grafana/data'
import { PrometheusDatasource } from '../../../datasource';
import { PrometheusLanguageProviderInterface } from '../../../language_provider';
import { EmptyLanguageProviderMock } from '../../../language_provider.mock';
import { getMockTimeRange } from '../../../test/__mocks__/datasource';
import { getMockTimeRange } from '../../../test/mocks/datasource';
import { PromOptions } from '../../../types';
import { PromVisualQuery } from '../../types';
packages/grafana-prometheus/src/querybuilder/shared/OperationList.test.tsx
+1 -1
View File
@@ -7,7 +7,7 @@ import { DataSourceApi, DataSourceInstanceSettings } from '@grafana/data';
import { PrometheusDatasource } from '../../datasource';
import { PrometheusLanguageProviderInterface } from '../../language_provider';
import { EmptyLanguageProviderMock } from '../../language_provider.mock';
import { getMockTimeRange } from '../../test/__mocks__/datasource';
import { getMockTimeRange } from '../../test/mocks/datasource';
import { PromOptions } from '../../types';
import { addOperationInQueryBuilder } from '../testUtils';
import { PromVisualQuery } from '../types';
packages/grafana-prometheus/src/test/__mocks__/datasource.ts → packages/grafana-prometheus/src/test/mocks/datasource.ts
View File
packages/grafana-ui/src/components/uPlot/utils.ts
+1 -7
View File
@@ -4,10 +4,8 @@ import {
DataFrame,
DisplayProcessor,
DisplayValue,
ensureTimeField,
Field,
fieldReducers,
FieldType,
getDisplayProcessor,
GrafanaTheme2,
reduceField,
@@ -197,11 +195,7 @@ export function preparePlotData2(
let vals = field.values;
if (i === 0) {
if (field.type === FieldType.time) {
data[0] = ensureTimeField(field).values;
} else {
data[0] = vals;
}
data[i] = vals;
return;
}
pkg/apis/secret/v0alpha1/keeper.go
+23 -3
View File
@@ -65,6 +65,22 @@ type KeeperSpec struct {
HashiCorp *HashiCorpKeeperConfig `json:"hashivault,omitempty"`
}
func (s *KeeperSpec) GetType() KeeperType {
if s.AWS != nil {
return AWSKeeperType
}
if s.Azure != nil {
return AzureKeeperType
}
if s.GCP != nil {
return GCPKeeperType
}
if s.HashiCorp != nil {
return HashiCorpKeeperType
}
return ""
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type KeeperList struct {
metav1.TypeMeta `json:",inline"`
@@ -102,9 +118,6 @@ type HashiCorpCredentials struct {
Token CredentialValue `json:"token"`
}
// Envelope encrytion details.
type Envelope struct{}
// Holds the way credentials are obtained.
// +union
type CredentialValue struct {
@@ -122,6 +135,13 @@ type CredentialValue struct {
ValueFromConfig string `json:"valueFromConfig,omitempty"`
}
// System Keeper.
type SystemKeeperConfig struct{}
func (s *SystemKeeperConfig) Type() KeeperType {
return "system"
}
// Remote Keepers.
type AWSKeeperConfig struct {
AWSCredentials `json:",inline"`
pkg/apis/secret/v0alpha1/zz_generated.deepcopy.go
+16 -16
View File
@@ -96,22 +96,6 @@ func (in *CredentialValue) DeepCopy() *CredentialValue {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Envelope) DeepCopyInto(out *Envelope) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Envelope.
func (in *Envelope) DeepCopy() *Envelope {
if in == nil {
return nil
}
out := new(Envelope)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GCPCredentials) DeepCopyInto(out *GCPCredentials) {
*out = *in
@@ -398,3 +382,19 @@ func (in *SecureValueStatus) DeepCopy() *SecureValueStatus {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SystemKeeperConfig) DeepCopyInto(out *SystemKeeperConfig) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SystemKeeperConfig.
func (in *SystemKeeperConfig) DeepCopy() *SystemKeeperConfig {
if in == nil {
return nil
}
out := new(SystemKeeperConfig)
in.DeepCopyInto(out)
return out
}
pkg/apis/secret/v0alpha1/zz_generated.openapi.go
+12 -12
View File
@@ -20,7 +20,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.AzureCredentials": schema_pkg_apis_secret_v0alpha1_AzureCredentials(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.AzureKeeperConfig": schema_pkg_apis_secret_v0alpha1_AzureKeeperConfig(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.CredentialValue": schema_pkg_apis_secret_v0alpha1_CredentialValue(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.Envelope": schema_pkg_apis_secret_v0alpha1_Envelope(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.GCPCredentials": schema_pkg_apis_secret_v0alpha1_GCPCredentials(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.GCPKeeperConfig": schema_pkg_apis_secret_v0alpha1_GCPKeeperConfig(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.HashiCorpCredentials": schema_pkg_apis_secret_v0alpha1_HashiCorpCredentials(ref),
@@ -33,6 +32,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.SecureValueList": schema_pkg_apis_secret_v0alpha1_SecureValueList(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.SecureValueSpec": schema_pkg_apis_secret_v0alpha1_SecureValueSpec(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.SecureValueStatus": schema_pkg_apis_secret_v0alpha1_SecureValueStatus(ref),
"github.com/grafana/grafana/pkg/apis/secret/v0alpha1.SystemKeeperConfig": schema_pkg_apis_secret_v0alpha1_SystemKeeperConfig(ref),
}
}
@@ -235,17 +235,6 @@ func schema_pkg_apis_secret_v0alpha1_CredentialValue(ref common.ReferenceCallbac
}
}
func schema_pkg_apis_secret_v0alpha1_Envelope(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "Envelope encrytion details.",
Type: []string{"object"},
},
},
}
}
func schema_pkg_apis_secret_v0alpha1_GCPCredentials(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
@@ -771,3 +760,14 @@ func schema_pkg_apis_secret_v0alpha1_SecureValueStatus(ref common.ReferenceCallb
},
}
}
func schema_pkg_apis_secret_v0alpha1_SystemKeeperConfig(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "System Keeper.",
Type: []string{"object"},
},
},
}
}
pkg/registry/apis/apis.go
+1 -1
View File
@@ -27,9 +27,9 @@ func ProvideRegistryServiceSink(
_ *iam.IdentityAccessManagementAPIBuilder,
_ *query.QueryAPIBuilder,
_ *userstorage.UserStorageAPIBuilder,
_ *secret.SecretAPIBuilder,
_ *provisioning.APIBuilder,
_ *ofrep.APIBuilder,
_ *secret.DependencyRegisterer,
) *Service {
return &Service{}
}
pkg/registry/apis/secret/accesscontrol.go
+5 -5
View File
@@ -27,12 +27,12 @@ var (
ScopeAllKeepers = ScopeProviderSecretKeepers.GetResourceAllScope()
)
func RegisterAccessControlRoles(service accesscontrol.Service) error {
func registerAccessControlRoles(service accesscontrol.Service) error {
// SecureValues
secureValuesReader := accesscontrol.RoleRegistration{
Role: accesscontrol.RoleDTO{
Name: "fixed:secret.securevalues:reader",
DisplayName: "Secrets Manager secure values reader",
DisplayName: "Secure Values Reader",
Description: "Read and list secure values.",
Group: "Secrets Manager",
Permissions: []accesscontrol.Permission{
@@ -48,7 +48,7 @@ func RegisterAccessControlRoles(service accesscontrol.Service) error {
secureValuesWriter := accesscontrol.RoleRegistration{
Role: accesscontrol.RoleDTO{
Name: "fixed:secret.securevalues:writer",
DisplayName: "Secrets Manager secure values writer",
DisplayName: "Secure Values Writer",
Description: "Create, update and delete secure values.",
Group: "Secrets Manager",
Permissions: []accesscontrol.Permission{
@@ -77,7 +77,7 @@ func RegisterAccessControlRoles(service accesscontrol.Service) error {
keepersReader := accesscontrol.RoleRegistration{
Role: accesscontrol.RoleDTO{
Name: "fixed:secret.keepers:reader",
DisplayName: "Secrets Manager keepers reader",
DisplayName: "Keepers Reader",
Description: "Read and list keepers.",
Group: "Secrets Manager",
Permissions: []accesscontrol.Permission{
@@ -93,7 +93,7 @@ func RegisterAccessControlRoles(service accesscontrol.Service) error {
keepersWriter := accesscontrol.RoleRegistration{
Role: accesscontrol.RoleDTO{
Name: "fixed:secret.keepers:writer",
DisplayName: "Secrets Manager keepers writer",
DisplayName: "Keepers Writer",
Description: "Create, update and delete keepers.",
Group: "Secrets Manager",
Permissions: []accesscontrol.Permission{
pkg/registry/apis/secret/decrypt/authorizer.go
+19 -2
View File
@@ -6,6 +6,8 @@ import (
"github.com/grafana/authlib/authn"
claims "github.com/grafana/authlib/types"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
@@ -13,17 +15,32 @@ import (
// decryptAuthorizer is the authorizer implementation for decrypt operations.
type decryptAuthorizer struct {
tracer trace.Tracer
allowList contracts.DecryptAllowList
}
func ProvideDecryptAuthorizer(allowList contracts.DecryptAllowList) contracts.DecryptAuthorizer {
func ProvideDecryptAuthorizer(tracer trace.Tracer, allowList contracts.DecryptAllowList) contracts.DecryptAuthorizer {
return &decryptAuthorizer{
tracer: tracer,
allowList: allowList,
}
}
// authorize checks whether the auth info token has the right permissions to decrypt the secure value.
func (a *decryptAuthorizer) Authorize(ctx context.Context, secureValueName string, secureValueDecrypters []string) (string, bool) {
func (a *decryptAuthorizer) Authorize(ctx context.Context, secureValueName string, secureValueDecrypters []string) (id string, isAllowed bool) {
ctx, span := a.tracer.Start(ctx, "DecryptAuthorizer.Authorize", trace.WithAttributes(
attribute.String("name", secureValueName),
attribute.StringSlice("decrypters", secureValueDecrypters),
))
defer span.End()
defer func() {
if id != "" {
span.SetAttributes(attribute.String("serviceIdentity", id))
}
span.SetAttributes(attribute.Bool("allowed", isAllowed))
}()
authInfo, ok := claims.AuthInfoFrom(ctx)
if !ok {
return "", false
pkg/registry/apis/secret/decrypt/authorizer_test.go
+20 -17
View File
@@ -7,14 +7,17 @@ import (
"github.com/grafana/authlib/authn"
"github.com/grafana/authlib/types"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"github.com/grafana/grafana/pkg/apimachinery/identity"
)
func TestDecryptAuthorizer(t *testing.T) {
tracer := noop.NewTracerProvider().Tracer("test")
t.Run("when no auth info is present, it returns false", func(t *testing.T) {
ctx := context.Background()
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
identity, allowed := authorizer.Authorize(ctx, "", nil)
require.Empty(t, identity)
@@ -23,7 +26,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when token permissions are empty, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{})
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
identity, allowed := authorizer.Authorize(ctx, "", nil)
require.NotEmpty(t, identity)
@@ -32,7 +35,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when service identity is empty, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "", []string{})
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
identity, allowed := authorizer.Authorize(ctx, "", nil)
require.Empty(t, identity)
@@ -40,7 +43,7 @@ func TestDecryptAuthorizer(t *testing.T) {
})
t.Run("when permission format is malformed (missing verb), it returns false", func(t *testing.T) {
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
// nameless
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues"})
@@ -56,7 +59,7 @@ func TestDecryptAuthorizer(t *testing.T) {
})
t.Run("when permission verb is not exactly `decrypt`, it returns false", func(t *testing.T) {
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
// nameless
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues:*"})
@@ -73,7 +76,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when permission does not have 2 or 3 parts, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app:decrypt"})
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
identity, allowed := authorizer.Authorize(ctx, "", nil)
require.NotEmpty(t, identity)
@@ -82,7 +85,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when permission has group that is not `secret.grafana.app`, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{"wrong.group/securevalues/invalid:decrypt"})
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
identity, allowed := authorizer.Authorize(ctx, "", nil)
require.NotEmpty(t, identity)
@@ -90,7 +93,7 @@ func TestDecryptAuthorizer(t *testing.T) {
})
t.Run("when permission has resource that is not `securevalues`, it returns false", func(t *testing.T) {
authorizer := ProvideDecryptAuthorizer(nil)
authorizer := ProvideDecryptAuthorizer(tracer, nil)
// nameless
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/invalid-resource:decrypt"})
@@ -107,7 +110,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when the identity is not in the allow list, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues:decrypt"})
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"allowed1": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"allowed1": {}})
identity, allowed := authorizer.Authorize(ctx, "", nil)
require.NotEmpty(t, identity)
@@ -115,7 +118,7 @@ func TestDecryptAuthorizer(t *testing.T) {
})
t.Run("when the identity doesn't match any allowed decrypters, it returns false", func(t *testing.T) {
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
// nameless
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues:decrypt"})
@@ -131,7 +134,7 @@ func TestDecryptAuthorizer(t *testing.T) {
})
t.Run("when the identity matches an allowed decrypter, it returns true", func(t *testing.T) {
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
// nameless
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues:decrypt"})
@@ -154,7 +157,7 @@ func TestDecryptAuthorizer(t *testing.T) {
"wrong.group/securevalues/group2:decrypt",
"secret.grafana.app/securevalues/identity:decrypt", // old style of identity+permission
})
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
identity, allowed := authorizer.Authorize(ctx, "name1", []string{"identity"})
require.True(t, allowed)
@@ -167,7 +170,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when empty secure value name with specific permission, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues/name:decrypt"})
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
identity, allowed := authorizer.Authorize(ctx, "", []string{"identity"})
require.Equal(t, "identity", identity)
@@ -176,7 +179,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when permission has an extra / but no name, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues/:decrypt"})
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
identity, allowed := authorizer.Authorize(ctx, "", []string{"identity"})
require.Equal(t, "identity", identity)
@@ -185,7 +188,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when the decrypters list is empty, meaning nothing can decrypt the secure value, it returns false", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity", []string{"secret.grafana.app/securevalues:decrypt"})
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
identity, allowed := authorizer.Authorize(ctx, "name", []string{})
require.Equal(t, "identity", identity)
@@ -194,7 +197,7 @@ func TestDecryptAuthorizer(t *testing.T) {
t.Run("when one of decrypters matches the identity, it returns true", func(t *testing.T) {
ctx := createAuthContext(context.Background(), "identity1", []string{"secret.grafana.app/securevalues:decrypt"})
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity1": {}, "identity2": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity1": {}, "identity2": {}})
identity, allowed := authorizer.Authorize(ctx, "", []string{"identity1", "identity2", "identity3"})
require.Equal(t, "identity1", identity)
@@ -202,7 +205,7 @@ func TestDecryptAuthorizer(t *testing.T) {
})
t.Run("permissions must be case-sensitive and return false", func(t *testing.T) {
authorizer := ProvideDecryptAuthorizer(map[string]struct{}{"identity": {}})
authorizer := ProvideDecryptAuthorizer(tracer, map[string]struct{}{"identity": {}})
ctx := createAuthContext(context.Background(), "identity", []string{"SECRET.grafana.app/securevalues:decrypt"})
identity, allowed := authorizer.Authorize(ctx, "", []string{"identity"})
pkg/registry/apis/secret/encryption/cipher/service/service.go
+5 -5
View File
@@ -7,9 +7,9 @@ import (
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher"
@@ -24,7 +24,7 @@ const (
// Service must not be used for cipher.
// Use secrets.Service implementing envelope encryption instead.
type Service struct {
tracer tracing.Tracer
tracer trace.Tracer
log log.Logger
cfg *setting.Cfg
@@ -35,7 +35,7 @@ type Service struct {
}
func NewEncryptionService(
tracer tracing.Tracer,
tracer trace.Tracer,
usageMetrics usagestats.Service,
cfg *setting.Cfg,
) (*Service, error) {
@@ -101,7 +101,7 @@ func (s *Service) registerUsageMetrics() {
}
func (s *Service) Decrypt(ctx context.Context, payload []byte, secret string) ([]byte, error) {
ctx, span := s.tracer.Start(ctx, "cipher.service.Decrypt")
ctx, span := s.tracer.Start(ctx, "CipherService.Decrypt")
defer span.End()
var err error
@@ -163,7 +163,7 @@ func (s *Service) deriveEncryptionAlgorithm(payload []byte) (string, []byte, err
}
func (s *Service) Encrypt(ctx context.Context, payload []byte, secret string) ([]byte, error) {
ctx, span := s.tracer.Start(ctx, "cipher.service.Encrypt")
ctx, span := s.tracer.Start(ctx, "CipherService.Encrypt")
defer span.End()
var err error
pkg/registry/apis/secret/encryption/cipher/service/service_test.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/registry/apis/secret/encryption/cipher"
"github.com/grafana/grafana/pkg/setting"
@@ -29,7 +29,7 @@ func newGcmService(t *testing.T) *Service {
},
}
svc, err := NewEncryptionService(tracing.InitializeTracerForTest(), usageStats, settings)
svc, err := NewEncryptionService(noop.NewTracerProvider().Tracer("test"), usageStats, settings)
require.NoError(t, err, "failed to set up encryption service")
return svc
}
pkg/registry/apis/secret/register.go
+17 -245
View File
@@ -4,268 +4,40 @@ import (
"context"
"fmt"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/kube-openapi/pkg/common"
claims "github.com/grafana/authlib/types"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/reststorage"
"github.com/grafana/grafana/pkg/services/accesscontrol"
authsvc "github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer"
"github.com/grafana/grafana/pkg/services/apiserver/builder"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
var (
_ builder.APIGroupBuilder = (*SecretAPIBuilder)(nil)
_ builder.APIGroupMutation = (*SecretAPIBuilder)(nil)
_ builder.APIGroupValidation = (*SecretAPIBuilder)(nil)
)
// DependencyRegisterer is set to satisfy wire gen and make sure the `RegisterDependencies` is called.
type DependencyRegisterer struct{}
type SecretAPIBuilder struct {
tracer tracing.Tracer
secureValueMetadataStorage contracts.SecureValueMetadataStorage
keeperMetadataStorage contracts.KeeperMetadataStorage
accessClient claims.AccessClient
decryptersAllowList map[string]struct{}
}
func NewSecretAPIBuilder(
tracer tracing.Tracer,
secureValueMetadataStorage contracts.SecureValueMetadataStorage,
keeperMetadataStorage contracts.KeeperMetadataStorage,
accessClient claims.AccessClient,
decryptersAllowList map[string]struct{},
) *SecretAPIBuilder {
return &SecretAPIBuilder{tracer, secureValueMetadataStorage, keeperMetadataStorage, accessClient, decryptersAllowList}
}
func RegisterAPIService(
func RegisterDependencies(
features featuremgmt.FeatureToggles,
cfg *setting.Cfg,
apiregistration builder.APIRegistrar,
tracer tracing.Tracer,
secureValueMetadataStorage contracts.SecureValueMetadataStorage,
keeperMetadataStorage contracts.KeeperMetadataStorage,
accessClient claims.AccessClient,
accessControlService accesscontrol.Service,
secretDBMigrator contracts.SecretDBMigrator,
) (*SecretAPIBuilder, error) {
// Skip registration unless opting into experimental apis and the secrets management app platform flag.
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) ||
!features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
accessControlService accesscontrol.Service,
) (*DependencyRegisterer, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) || !features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
return nil, nil
}
// Some DBs that claim to be MySQL/Postgres-compatible might not support table locking.
lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true)
if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil {
return nil, fmt.Errorf("running secret database migrations: %w", err)
// Permissions for requests in multi-tenant mode will come from HG.
if err := registerAccessControlRoles(accessControlService); err != nil {
return nil, fmt.Errorf("registering access control roles: %w", err)
}
if err := RegisterAccessControlRoles(accessControlService); err != nil {
return nil, fmt.Errorf("register secret access control roles: %w", err)
}
// We shouldn't need to create the DB in HG, as that will use the MT api server.
if cfg.StackID == "" {
// Some DBs that claim to be MySQL/Postgres-compatible might not support table locking.
lockDatabase := cfg.Raw.Section("database").Key("migration_locking").MustBool(true)
builder := NewSecretAPIBuilder(
tracer,
secureValueMetadataStorage,
keeperMetadataStorage,
accessClient,
nil, // OSS does not need an allow list.
)
apiregistration.RegisterAPI(builder)
return builder, nil
}
// GetGroupVersion returns the tuple of `group` and `version` for the API which uniquely identifies it.
func (b *SecretAPIBuilder) GetGroupVersion() schema.GroupVersion {
return secretv0alpha1.SchemeGroupVersion
}
// InstallSchema is called by the `apiserver` which exposes the defined kinds.
func (b *SecretAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
err := secretv0alpha1.AddKnownTypes(scheme, secretv0alpha1.VERSION)
if err != nil {
return err
}
// Link this version to the internal representation.
// This is used for server-side-apply (PATCH), and avoids the error:
// "no kind is registered for the type"
err = secretv0alpha1.AddKnownTypes(scheme, runtime.APIVersionInternal)
if err != nil {
return err
}
// Internal Kubernetes metadata API. Presumably to display the available APIs?
// e.g. http://localhost:3000/apis/secret.grafana.app/v0alpha1
metav1.AddToGroupVersion(scheme, secretv0alpha1.SchemeGroupVersion)
// This sets the priority in case we have multiple versions.
// By default Kubernetes will only let you use `kubectl get <resource>` with one version.
// In case there are multiple versions, we'd need to pass the full path with the `--raw` flag.
if err := scheme.SetVersionPriority(secretv0alpha1.SchemeGroupVersion); err != nil {
return fmt.Errorf("scheme set version priority: %w", err)
}
return nil
}
func (b *SecretAPIBuilder) AllowedV0Alpha1Resources() []string {
return nil
}
// UpdateAPIGroupInfo is called when creating a generic API server for this group of kinds.
func (b *SecretAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error {
secureValueResource := secretv0alpha1.SecureValuesResourceInfo
keeperResource := secretv0alpha1.KeeperResourceInfo
// rest.Storage is a generic interface for RESTful storage services.
// The constructors need to at least implement this interface, but will most likely implement
// other interfaces that equal to different operations like `get`, `list` and so on.
secureRestStorage := map[string]rest.Storage{
// Default path for `securevalue`.
// The `reststorage.SecureValueRest` struct will implement interfaces for CRUDL operations on `securevalue`.
secureValueResource.StoragePath(): reststorage.NewSecureValueRest(b.secureValueMetadataStorage, secureValueResource),
// The `reststorage.KeeperRest` struct will implement interfaces for CRUDL operations on `keeper`.
keeperResource.StoragePath(): reststorage.NewKeeperRest(b.keeperMetadataStorage, b.accessClient, keeperResource),
}
apiGroupInfo.VersionedResourcesStorageMap[secretv0alpha1.VERSION] = secureRestStorage
return nil
}
// GetOpenAPIDefinitions, is this only for documentation?
func (b *SecretAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions {
return secretv0alpha1.GetOpenAPIDefinitions
}
// GetAuthorizer decides whether the request is allowed, denied or no opinion based on credentials and request attributes.
// Usually most resource are stored in folders (e.g. alerts, dashboards), which allows users to manage permissions at folder level,
// rather than at resource level which also has the benefit of lowering the load on AuthZ side, since instead of storing access to
// a single dashboard, you'd store access to all dashboards in a specific folder.
// For Secrets, this is not the case, but if we want to make it so, we need to update this ResourceAuthorizer to check the containing folder.
// If we ever want to do that, get guidance from IAM first as well.
func (b *SecretAPIBuilder) GetAuthorizer() authorizer.Authorizer {
return authsvc.NewResourceAuthorizer(b.accessClient)
}
// Validate is called in `Create`, `Update` and `Delete` REST funcs, if the body calls the argument `rest.ValidateObjectFunc`.
func (b *SecretAPIBuilder) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
obj := a.GetObject()
operation := a.GetOperation()
if obj == nil || operation == admission.Connect {
return nil // This is normal for sub-resource
}
groupKind := obj.GetObjectKind().GroupVersionKind().GroupKind()
// Generic validations for all kinds. At this point the name+namespace must not be empty.
if a.GetName() == "" {
return apierrors.NewInvalid(
groupKind,
a.GetName(),
field.ErrorList{field.Required(field.NewPath("metadata", "name"), "a `name` is required")},
)
}
if a.GetNamespace() == "" {
return apierrors.NewInvalid(
groupKind,
a.GetName(),
field.ErrorList{field.Required(field.NewPath("metadata", "namespace"), "a `namespace` is required")},
)
}
switch typedObj := obj.(type) {
case *secretv0alpha1.SecureValue:
var oldObj *secretv0alpha1.SecureValue
if a.GetOldObject() != nil {
var ok bool
oldObj, ok = a.GetOldObject().(*secretv0alpha1.SecureValue)
if !ok {
return apierrors.NewBadRequest(fmt.Sprintf("old object is not a SecureValue, found %T", a.GetOldObject()))
}
}
if errs := reststorage.ValidateSecureValue(typedObj, oldObj, operation, b.decryptersAllowList); len(errs) > 0 {
return apierrors.NewInvalid(groupKind, a.GetName(), errs)
}
return nil
case *secretv0alpha1.Keeper:
if errs := reststorage.ValidateKeeper(typedObj, operation); len(errs) > 0 {
return apierrors.NewInvalid(groupKind, a.GetName(), errs)
}
return nil
}
return apierrors.NewBadRequest(fmt.Sprintf("unknown spec %T", obj))
}
func (b *SecretAPIBuilder) Mutate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
obj := a.GetObject()
operation := a.GetOperation()
if obj == nil || operation == admission.Connect {
return nil // This is normal for sub-resource
}
// When creating a resource and the name is empty, we need to generate one.
if operation == admission.Create && a.GetName() == "" {
generatedName, err := util.GetRandomString(8)
if err != nil {
return fmt.Errorf("generate random string: %w", err)
}
switch typedObj := obj.(type) {
case *secretv0alpha1.SecureValue:
optionalPrefix := typedObj.GenerateName
if optionalPrefix == "" {
optionalPrefix = "sv-"
}
typedObj.Name = optionalPrefix + generatedName
case *secretv0alpha1.Keeper:
optionalPrefix := typedObj.GenerateName
if optionalPrefix == "" {
optionalPrefix = "kp-"
}
typedObj.Name = optionalPrefix + generatedName
// This is needed to wire up and run DB migrations for Secrets Manager, which is not run by the generic OSS DB migrator.
if err := secretDBMigrator.RunMigrations(context.Background(), lockDatabase); err != nil {
return nil, fmt.Errorf("running secret database migrations: %w", err)
}
}
// On any mutation to a `SecureValue`, override the `phase` as `Pending` and an empty `message`.
if operation == admission.Create || operation == admission.Update {
sv, ok := obj.(*secretv0alpha1.SecureValue)
if ok && sv != nil {
sv.Status.Phase = secretv0alpha1.SecureValuePhasePending
sv.Status.Message = ""
}
}
return nil
return &DependencyRegisterer{}, nil
}
pkg/registry/apis/secret/reststorage/keeper_rest.go
-386
View File
@@ -1,386 +0,0 @@
package reststorage
import (
"context"
"errors"
"fmt"
"strings"
claims "github.com/grafana/authlib/types"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"github.com/grafana/grafana/pkg/apimachinery/utils"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
)
var (
_ rest.Scoper = (*KeeperRest)(nil)
_ rest.SingularNameProvider = (*KeeperRest)(nil)
_ rest.Getter = (*KeeperRest)(nil)
_ rest.Lister = (*KeeperRest)(nil)
_ rest.Storage = (*KeeperRest)(nil)
_ rest.Creater = (*KeeperRest)(nil)
_ rest.Updater = (*KeeperRest)(nil)
_ rest.GracefulDeleter = (*KeeperRest)(nil)
)
// KeeperRest is an implementation of CRUDL operations on a `keeper` backed by TODO.
type KeeperRest struct {
storage contracts.KeeperMetadataStorage
accessClient claims.AccessClient
resource utils.ResourceInfo
tableConverter rest.TableConvertor
}
// NewKeeperRest is a returns a constructed `*KeeperRest`.
func NewKeeperRest(storage contracts.KeeperMetadataStorage, accessClient claims.AccessClient, resource utils.ResourceInfo) *KeeperRest {
return &KeeperRest{storage, accessClient, resource, resource.TableConverter()}
}
// New returns an empty `*Keeper` that is used by the `Create` method.
func (s *KeeperRest) New() runtime.Object {
return s.resource.NewFunc()
}
// Destroy is called when? [TODO]
func (s *KeeperRest) Destroy() {}
// NamespaceScoped returns `true` because the storage is namespaced (== org).
func (s *KeeperRest) NamespaceScoped() bool {
return true
}
// GetSingularName is used by `kubectl` discovery to have singular name representation of resources.
func (s *KeeperRest) GetSingularName() string {
return s.resource.GetSingularName()
}
// NewList returns an empty `*KeeperList` that is used by the `List` method.
func (s *KeeperRest) NewList() runtime.Object {
return s.resource.NewListFunc()
}
// ConvertToTable is used by Kubernetes and converts objects to `metav1.Table`.
func (s *KeeperRest) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
return s.tableConverter.ConvertToTable(ctx, object, tableOptions)
}
// List calls the inner `store` (persistence) and returns a list of `Keepers` within a `namespace` filtered by the `options`.
func (s *KeeperRest) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
namespace, ok := request.NamespaceFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing namespace")
}
user, ok := claims.AuthInfoFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing auth info in context")
}
hasPermissionFor, err := s.accessClient.Compile(ctx, user, claims.ListRequest{
Group: secretv0alpha1.GROUP,
Resource: secretv0alpha1.KeeperResourceInfo.GetName(),
Namespace: namespace,
Verb: utils.VerbGet,
})
if err != nil {
return nil, fmt.Errorf("failed to compile checker: %w", err)
}
labelSelector := options.LabelSelector
if labelSelector == nil {
labelSelector = labels.Everything()
}
keepersList, err := s.storage.List(ctx, xkube.Namespace(namespace))
if err != nil {
return nil, fmt.Errorf("failed to list keepers: %w", err)
}
allowedKeepers := make([]secretv0alpha1.Keeper, 0)
for _, keeper := range keepersList {
// Check whether the user has permission to access this specific Keeper in the namespace.
if !hasPermissionFor(keeper.Name, "") {
continue
}
if labelSelector.Matches(labels.Set(keeper.Labels)) {
allowedKeepers = append(allowedKeepers, keeper)
}
}
return &secretv0alpha1.KeeperList{
Items: allowedKeepers,
}, nil
}
// Get calls the inner `store` (persistence) and returns a `Keeper` by `name`.
func (s *KeeperRest) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
namespace, ok := request.NamespaceFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing namespace")
}
// TODO: readopts
kp, err := s.storage.Read(ctx, xkube.Namespace(namespace), name, contracts.ReadOpts{})
if err != nil {
if errors.Is(err, contracts.ErrKeeperNotFound) {
return nil, s.resource.NewNotFound(name)
}
return nil, fmt.Errorf("failed to read keeper: %w", err)
}
return kp, nil
}
// Create a new `Keeper`. Does some validation and allows empty `name` (generated).
func (s *KeeperRest) Create(
ctx context.Context,
obj runtime.Object,
createValidation rest.ValidateObjectFunc,
options *metav1.CreateOptions,
) (runtime.Object, error) {
kp, ok := obj.(*secretv0alpha1.Keeper)
if !ok {
return nil, fmt.Errorf("expected Keeper for create")
}
user, ok := claims.AuthInfoFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing auth info in context")
}
if err := createValidation(ctx, obj); err != nil {
return nil, err
}
createdKeeper, err := s.storage.Create(ctx, kp, user.GetUID())
if err != nil {
var kErr xkube.ErrorLister
if errors.As(err, &kErr) {
return nil, apierrors.NewInvalid(kp.GroupVersionKind().GroupKind(), kp.Name, kErr.ErrorList())
}
return nil, fmt.Errorf("failed to create keeper: %w", err)
}
return createdKeeper, nil
}
// Update a `Keeper`'s `value`. The second return parameter indicates whether the resource was newly created.
func (s *KeeperRest) Update(
ctx context.Context,
name string,
objInfo rest.UpdatedObjectInfo,
createValidation rest.ValidateObjectFunc,
updateValidation rest.ValidateObjectUpdateFunc,
forceAllowCreate bool,
options *metav1.UpdateOptions,
) (runtime.Object, bool, error) {
user, ok := claims.AuthInfoFrom(ctx)
if !ok {
return nil, false, fmt.Errorf("missing auth info in context")
}
oldObj, err := s.Get(ctx, name, &metav1.GetOptions{})
if err != nil {
return nil, false, err
}
// Makes sure the UID and ResourceVersion are OK.
// TODO: this also makes it so the labels and annotations are additive, unless we check and remove manually.
newObj, err := objInfo.UpdatedObject(ctx, oldObj)
if err != nil {
return nil, false, fmt.Errorf("k8s updated object: %w", err)
}
// The current supported behavior for `Update` is to replace the entire `spec` with the new one.
// Each provider-specific setting of a keeper lives at the top-level, so it makes it possible to change a provider
// during an update. Otherwise both old and new providers would be merged in the `newObj` which is not allowed.
if err := updateValidation(ctx, newObj, oldObj); err != nil {
return nil, false, err
}
newKeeper, ok := newObj.(*secretv0alpha1.Keeper)
if !ok {
return nil, false, fmt.Errorf("expected Keeper for update")
}
// TODO: do we need to do this here again? Probably not, but double-check!
newKeeper.Annotations = xkube.CleanAnnotations(newKeeper.Annotations)
// Current implementation replaces everything passed in the spec, so it is not a PATCH. Do we want/need to support that?
updatedKeeper, err := s.storage.Update(ctx, newKeeper, user.GetUID())
if err != nil {
var kErr xkube.ErrorLister
if errors.As(err, &kErr) {
return nil, false, apierrors.NewInvalid(newKeeper.GroupVersionKind().GroupKind(), newKeeper.Name, kErr.ErrorList())
}
return nil, false, fmt.Errorf("failed to update keeper: %w", err)
}
return updatedKeeper, false, nil
}
// Delete calls the inner `store` (persistence) in order to delete the `Keeper`.
// The second return parameter `bool` indicates whether the delete was intant or not. It always is for `Keepers`.
func (s *KeeperRest) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
namespace, ok := request.NamespaceFrom(ctx)
if !ok {
return nil, false, fmt.Errorf("missing namespace")
}
err := s.storage.Delete(ctx, xkube.Namespace(namespace), name)
if err != nil {
if errors.Is(err, contracts.ErrKeeperNotFound) {
return nil, false, s.resource.NewNotFound(name)
}
return nil, false, fmt.Errorf("failed to delete keeper: %w", err)
}
return nil, true, nil
}
// ValidateKeeper does basic spec validation of a keeper.
func ValidateKeeper(keeper *secretv0alpha1.Keeper, operation admission.Operation) field.ErrorList {
// Only validate Create and Update for now.
if operation != admission.Create && operation != admission.Update {
return nil
}
errs := make(field.ErrorList, 0)
if keeper.Spec.Description == "" {
errs = append(errs, field.Required(field.NewPath("spec", "description"), "a `description` is required"))
}
// Only one keeper type can be configured. Return early and don't validate the specific keeper fields.
if err := validateKeepers(keeper); err != nil {
errs = append(errs, err)
return errs
}
if keeper.Spec.AWS != nil {
if err := validateCredentialValue(field.NewPath("spec", "aws", "accessKeyId"), keeper.Spec.AWS.AccessKeyID); err != nil {
errs = append(errs, err)
}
if err := validateCredentialValue(field.NewPath("spec", "aws", "secretAccessKey"), keeper.Spec.AWS.SecretAccessKey); err != nil {
errs = append(errs, err)
}
}
if keeper.Spec.Azure != nil {
if keeper.Spec.Azure.KeyVaultName == "" {
errs = append(errs, field.Required(field.NewPath("spec", "azure", "keyVaultName"), "a `keyVaultName` is required"))
}
if keeper.Spec.Azure.TenantID == "" {
errs = append(errs, field.Required(field.NewPath("spec", "azure", "tenantId"), "a `tenantId` is required"))
}
if keeper.Spec.Azure.ClientID == "" {
errs = append(errs, field.Required(field.NewPath("spec", "azure", "clientId"), "a `clientId` is required"))
}
if err := validateCredentialValue(field.NewPath("spec", "azure", "clientSecret"), keeper.Spec.Azure.ClientSecret); err != nil {
errs = append(errs, err)
}
}
if keeper.Spec.GCP != nil {
if keeper.Spec.GCP.ProjectID == "" {
errs = append(errs, field.Required(field.NewPath("spec", "gcp", "projectId"), "a `projectId` is required"))
}
if keeper.Spec.GCP.CredentialsFile == "" {
errs = append(errs, field.Required(field.NewPath("spec", "gcp", "credentialsFile"), "a `credentialsFile` is required"))
}
}
if keeper.Spec.HashiCorp != nil {
if keeper.Spec.HashiCorp.Address == "" {
errs = append(errs, field.Required(field.NewPath("spec", "hashicorp", "address"), "a `address` is required"))
}
if err := validateCredentialValue(field.NewPath("spec", "hashicorp", "token"), keeper.Spec.HashiCorp.Token); err != nil {
errs = append(errs, err)
}
}
return errs
}
func validateKeepers(keeper *secretv0alpha1.Keeper) *field.Error {
availableKeepers := map[string]bool{
"aws": keeper.Spec.AWS != nil,
"azure": keeper.Spec.Azure != nil,
"gcp": keeper.Spec.GCP != nil,
"hashicorp": keeper.Spec.HashiCorp != nil,
}
configuredKeepers := make([]string, 0)
for keeperKind, notNil := range availableKeepers {
if notNil {
configuredKeepers = append(configuredKeepers, keeperKind)
}
}
if len(configuredKeepers) == 0 {
return field.Required(field.NewPath("spec"), "at least one `keeper` must be present")
}
if len(configuredKeepers) > 1 {
return field.Invalid(
field.NewPath("spec"),
strings.Join(configuredKeepers, " & "),
"only one `keeper` can be present at a time but found more",
)
}
return nil
}
func validateCredentialValue(path *field.Path, credentials secretv0alpha1.CredentialValue) *field.Error {
availableOptions := map[string]bool{
"secureValueName": credentials.SecureValueName != "",
"valueFromEnv": credentials.ValueFromEnv != "",
"valueFromConfig": credentials.ValueFromConfig != "",
}
configuredCredentials := make([]string, 0)
for credentialKind, notEmpty := range availableOptions {
if notEmpty {
configuredCredentials = append(configuredCredentials, credentialKind)
}
}
if len(configuredCredentials) == 0 {
return field.Required(path, "one of `secureValueName`, `valueFromEnv` or `valueFromConfig` must be present")
}
if len(configuredCredentials) > 1 {
return field.Invalid(
path,
strings.Join(configuredCredentials, " & "),
"only one of `secureValueName`, `valueFromEnv` or `valueFromConfig` must be present at a time but found more",
)
}
return nil
}
pkg/registry/apis/secret/reststorage/keeper_rest_test.go
-276
View File
@@ -1,276 +0,0 @@
package reststorage
import (
"testing"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/stretchr/testify/require"
"k8s.io/apiserver/pkg/admission"
)
func TestValidateKeeper(t *testing.T) {
t.Run("when creating a new keeper", func(t *testing.T) {
t.Run("the `description` must be present", func(t *testing.T) {
keeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
AWS: &secretv0alpha1.AWSKeeperConfig{
AWSCredentials: secretv0alpha1.AWSCredentials{
AccessKeyID: secretv0alpha1.CredentialValue{ValueFromEnv: "some-value"},
SecretAccessKey: secretv0alpha1.CredentialValue{ValueFromEnv: "some-value"},
KMSKeyID: "kms-key-id",
},
},
},
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.description", errs[0].Field)
})
})
t.Run("only one `keeper` must be present", func(t *testing.T) {
keeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "short description",
AWS: &secretv0alpha1.AWSKeeperConfig{},
Azure: &secretv0alpha1.AzureKeeperConfig{},
GCP: &secretv0alpha1.GCPKeeperConfig{},
HashiCorp: &secretv0alpha1.HashiCorpKeeperConfig{},
},
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("at least one `keeper` must be present", func(t *testing.T) {
keeper := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "description",
},
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("aws keeper validation", func(t *testing.T) {
validKeeperAWS := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "description",
AWS: &secretv0alpha1.AWSKeeperConfig{
AWSCredentials: secretv0alpha1.AWSCredentials{
AccessKeyID: secretv0alpha1.CredentialValue{
ValueFromEnv: "some-value",
},
SecretAccessKey: secretv0alpha1.CredentialValue{
SecureValueName: "some-value",
},
KMSKeyID: "optional",
},
},
},
}
t.Run("`accessKeyId` must be present", func(t *testing.T) {
t.Run("at least one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperAWS.DeepCopy()
keeper.Spec.AWS.AccessKeyID = secretv0alpha1.CredentialValue{}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.aws.accessKeyId", errs[0].Field)
})
t.Run("at most one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperAWS.DeepCopy()
keeper.Spec.AWS.AccessKeyID = secretv0alpha1.CredentialValue{
SecureValueName: "a",
ValueFromEnv: "b",
ValueFromConfig: "c",
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.aws.accessKeyId", errs[0].Field)
})
})
t.Run("`secretAccessKey` must be present", func(t *testing.T) {
t.Run("at least one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperAWS.DeepCopy()
keeper.Spec.AWS.SecretAccessKey = secretv0alpha1.CredentialValue{}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.aws.secretAccessKey", errs[0].Field)
})
t.Run("at most one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperAWS.DeepCopy()
keeper.Spec.AWS.SecretAccessKey = secretv0alpha1.CredentialValue{
SecureValueName: "a",
ValueFromEnv: "b",
ValueFromConfig: "c",
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.aws.secretAccessKey", errs[0].Field)
})
})
})
t.Run("azure keeper validation", func(t *testing.T) {
validKeeperAzure := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "description",
Azure: &secretv0alpha1.AzureKeeperConfig{
AzureCredentials: secretv0alpha1.AzureCredentials{
KeyVaultName: "kv-name",
TenantID: "tenant-id",
ClientID: "client-id",
ClientSecret: secretv0alpha1.CredentialValue{
ValueFromConfig: "config.path.value",
},
},
},
},
}
t.Run("`keyVaultName` must be present", func(t *testing.T) {
keeper := validKeeperAzure.DeepCopy()
keeper.Spec.Azure.KeyVaultName = ""
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.azure.keyVaultName", errs[0].Field)
})
t.Run("`tenantId` must be present", func(t *testing.T) {
keeper := validKeeperAzure.DeepCopy()
keeper.Spec.Azure.TenantID = ""
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.azure.tenantId", errs[0].Field)
})
t.Run("`clientId` must be present", func(t *testing.T) {
keeper := validKeeperAzure.DeepCopy()
keeper.Spec.Azure.ClientID = ""
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.azure.clientId", errs[0].Field)
})
t.Run("`clientSecret` must be present", func(t *testing.T) {
t.Run("at least one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperAzure.DeepCopy()
keeper.Spec.Azure.ClientSecret = secretv0alpha1.CredentialValue{}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.azure.clientSecret", errs[0].Field)
})
t.Run("at most one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperAzure.DeepCopy()
keeper.Spec.Azure.ClientSecret = secretv0alpha1.CredentialValue{
SecureValueName: "a",
ValueFromEnv: "b",
ValueFromConfig: "c",
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.azure.clientSecret", errs[0].Field)
})
})
})
t.Run("gcp keeper validation", func(t *testing.T) {
validKeeperGCP := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "description",
GCP: &secretv0alpha1.GCPKeeperConfig{
GCPCredentials: secretv0alpha1.GCPCredentials{
ProjectID: "project-id",
CredentialsFile: "/path/to/credentials/file.json",
},
},
},
}
t.Run("`projectId` must be present", func(t *testing.T) {
keeper := validKeeperGCP.DeepCopy()
keeper.Spec.GCP.ProjectID = ""
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.gcp.projectId", errs[0].Field)
})
t.Run("`credentialsFile` must be present", func(t *testing.T) {
keeper := validKeeperGCP.DeepCopy()
keeper.Spec.GCP.CredentialsFile = ""
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.gcp.credentialsFile", errs[0].Field)
})
})
t.Run("hashicorp keeper validation", func(t *testing.T) {
validKeeperHashiCorp := &secretv0alpha1.Keeper{
Spec: secretv0alpha1.KeeperSpec{
Description: "description",
HashiCorp: &secretv0alpha1.HashiCorpKeeperConfig{
HashiCorpCredentials: secretv0alpha1.HashiCorpCredentials{
Address: "http://address",
Token: secretv0alpha1.CredentialValue{
ValueFromConfig: "config.path.value",
},
},
},
},
}
t.Run("`address` must be present", func(t *testing.T) {
keeper := validKeeperHashiCorp.DeepCopy()
keeper.Spec.HashiCorp.Address = ""
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.hashicorp.address", errs[0].Field)
})
t.Run("`token` must be present", func(t *testing.T) {
t.Run("at least one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperHashiCorp.DeepCopy()
keeper.Spec.HashiCorp.Token = secretv0alpha1.CredentialValue{}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.hashicorp.token", errs[0].Field)
})
t.Run("at most one of the credential value must be present", func(t *testing.T) {
keeper := validKeeperHashiCorp.DeepCopy()
keeper.Spec.HashiCorp.Token = secretv0alpha1.CredentialValue{
SecureValueName: "a",
ValueFromEnv: "b",
ValueFromConfig: "c",
}
errs := ValidateKeeper(keeper, admission.Create)
require.Len(t, errs, 1)
require.Equal(t, "spec.hashicorp.token", errs[0].Field)
})
})
})
}
pkg/registry/apis/secret/reststorage/secure_value_rest.go
-379
View File
@@ -1,379 +0,0 @@
package reststorage
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
claims "github.com/grafana/authlib/types"
"k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"github.com/grafana/grafana/pkg/apimachinery/utils"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
)
var (
_ rest.Scoper = (*SecureValueRest)(nil)
_ rest.SingularNameProvider = (*SecureValueRest)(nil)
_ rest.Getter = (*SecureValueRest)(nil)
_ rest.Lister = (*SecureValueRest)(nil)
_ rest.Storage = (*SecureValueRest)(nil)
_ rest.Creater = (*SecureValueRest)(nil)
_ rest.Updater = (*SecureValueRest)(nil)
_ rest.GracefulDeleter = (*SecureValueRest)(nil)
)
// SecureValueRest is an implementation of CRUDL operations on a `securevalue` backed by a persistence layer `store`.
type SecureValueRest struct {
storage contracts.SecureValueMetadataStorage
resource utils.ResourceInfo
tableConverter rest.TableConvertor
}
// NewSecureValueRest is a returns a constructed `*SecureValueRest`.
func NewSecureValueRest(storage contracts.SecureValueMetadataStorage, resource utils.ResourceInfo) *SecureValueRest {
return &SecureValueRest{storage, resource, resource.TableConverter()}
}
// New returns an empty `*SecureValue` that is used by the `Create` method.
func (s *SecureValueRest) New() runtime.Object {
return s.resource.NewFunc()
}
// Destroy is called when? [TODO]
func (s *SecureValueRest) Destroy() {}
// NamespaceScoped returns `true` because the storage is namespaced (== org).
func (s *SecureValueRest) NamespaceScoped() bool {
return true
}
// GetSingularName is used by `kubectl` discovery to have singular name representation of resources.
func (s *SecureValueRest) GetSingularName() string {
return s.resource.GetSingularName()
}
// NewList returns an empty `*SecureValueList` that is used by the `List` method.
func (s *SecureValueRest) NewList() runtime.Object {
return s.resource.NewListFunc()
}
// ConvertToTable is used by Kubernetes and converts objects to `metav1.Table`.
func (s *SecureValueRest) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
return s.tableConverter.ConvertToTable(ctx, object, tableOptions)
}
// List calls the inner `store` (persistence) and returns a list of `securevalues` within a `namespace` filtered by the `options`.
func (s *SecureValueRest) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
namespace, ok := request.NamespaceFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing namespace")
}
secureValueList, err := s.storage.List(ctx, xkube.Namespace(namespace))
if err != nil {
return nil, fmt.Errorf("failed to list secure values: %w", err)
}
labelSelector := options.LabelSelector
if labelSelector == nil {
labelSelector = labels.Everything()
}
fieldSelector := options.FieldSelector
if fieldSelector == nil {
fieldSelector = fields.Everything()
}
allowedSecureValues := make([]secretv0alpha1.SecureValue, 0, len(secureValueList))
for _, secureValue := range secureValueList {
// Filter by label
if labelSelector.Matches(labels.Set(secureValue.Labels)) {
// Filter by status.phase
if fieldSelector.Matches(fields.Set{"status.phase": string(secureValue.Status.Phase)}) {
allowedSecureValues = append(allowedSecureValues, secureValue)
}
}
}
return &secretv0alpha1.SecureValueList{Items: allowedSecureValues}, nil
}
// Get calls the inner `store` (persistence) and returns a `securevalue` by `name`. It will NOT return the decrypted `value`.
func (s *SecureValueRest) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
namespace, ok := request.NamespaceFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing namespace")
}
sv, err := s.storage.Read(ctx, xkube.Namespace(namespace), name, contracts.ReadOpts{})
if err != nil {
if errors.Is(err, contracts.ErrSecureValueNotFound) {
return nil, s.resource.NewNotFound(name)
}
return nil, fmt.Errorf("failed to read secure value: %w", err)
}
return sv, nil
}
// Create a new `securevalue`. Does some validation and allows empty `name` (generated).
func (s *SecureValueRest) Create(
ctx context.Context,
obj runtime.Object,
createValidation rest.ValidateObjectFunc,
_ *metav1.CreateOptions,
) (runtime.Object, error) {
sv, ok := obj.(*secretv0alpha1.SecureValue)
if !ok {
return nil, fmt.Errorf("expected SecureValue for create")
}
if err := createValidation(ctx, obj); err != nil {
return nil, err
}
user, ok := claims.AuthInfoFrom(ctx)
if !ok {
return nil, fmt.Errorf("missing auth info in context")
}
createdSecureValueMetadata, err := s.storage.Create(ctx, sv, user.GetUID())
if err != nil {
return nil, fmt.Errorf("creating secure value %+w", err)
}
return createdSecureValueMetadata, nil
}
// Update a `securevalue`'s `value`. The second return parameter indicates whether the resource was newly created.
// Currently does not support "create on update" functionality. If the securevalue does not yet exist, it returns an error.
func (s *SecureValueRest) Update(
ctx context.Context,
name string,
objInfo rest.UpdatedObjectInfo,
_ rest.ValidateObjectFunc,
updateValidation rest.ValidateObjectUpdateFunc,
_forceAllowCreate bool,
_ *metav1.UpdateOptions,
) (runtime.Object, bool, error) {
oldObj, err := s.Get(ctx, name, &metav1.GetOptions{})
if err != nil {
return nil, false, err
}
// Makes sure the UID and ResourceVersion are OK.
// TODO: this also makes it so the labels and annotations are additive, unless we check and remove manually.
newObj, err := objInfo.UpdatedObject(ctx, oldObj)
if err != nil {
return nil, false, fmt.Errorf("k8s updated object: %w", err)
}
if err := updateValidation(ctx, newObj, oldObj); err != nil {
return nil, false, err
}
newSecureValue, ok := newObj.(*secretv0alpha1.SecureValue)
if !ok {
return nil, false, fmt.Errorf("expected SecureValue for update")
}
// TODO: do we need to do this here again? Probably not, but double-check!
newSecureValue.Annotations = xkube.CleanAnnotations(newSecureValue.Annotations)
user, ok := claims.AuthInfoFrom(ctx)
if !ok {
return nil, false, fmt.Errorf("missing auth info in context")
}
// Current implementation replaces everything passed in the spec, so it is not a PATCH. Do we want/need to support that?
updatedSecureValueMetadata, err := s.storage.Update(ctx, newSecureValue, user.GetUID())
if err != nil {
return updatedSecureValueMetadata, false, fmt.Errorf("updating secure value metadata: %+w", err)
}
return updatedSecureValueMetadata, false, nil
}
// The second return parameter `bool` indicates whether the delete was instant or not. It always is for `securevalues`.
func (s *SecureValueRest) Delete(ctx context.Context, name string, _ rest.ValidateObjectFunc, _ *metav1.DeleteOptions) (runtime.Object, bool, error) {
namespace, ok := request.NamespaceFrom(ctx)
if !ok {
return nil, false, fmt.Errorf("missing namespace")
}
if err := s.storage.Delete(ctx, xkube.Namespace(namespace), name); err != nil {
if errors.Is(err, contracts.ErrSecureValueNotFound) {
return nil, false, s.resource.NewNotFound(name)
}
return nil, false, fmt.Errorf("deleting secure value: %+w", err)
}
return nil, false, nil
}
// ValidateSecureValue does basic spec validation of a securevalue.
func ValidateSecureValue(sv, oldSv *secretv0alpha1.SecureValue, operation admission.Operation, decryptersAllowList map[string]struct{}) field.ErrorList {
errs := make(field.ErrorList, 0)
// Operation-specific field validation.
switch operation {
case admission.Create:
errs = validateSecureValueCreate(sv)
// If we plan to support PATCH-style updates, we shouldn't be requiring fields to be set.
case admission.Update:
errs = validateSecureValueUpdate(sv, oldSv)
case admission.Delete:
case admission.Connect:
}
// General validations.
if len(sv.Spec.Value) > contracts.SECURE_VALUE_RAW_INPUT_MAX_SIZE_BYTES {
errs = append(
errs,
field.TooLong(field.NewPath("spec", "value"), len(sv.Spec.Value), contracts.SECURE_VALUE_RAW_INPUT_MAX_SIZE_BYTES),
)
}
if errs := validateDecrypters(sv.Spec.Decrypters, decryptersAllowList); len(errs) > 0 {
return errs
}
return errs
}
// validateSecureValueCreate does basic spec validation of a securevalue for the Create operation.
func validateSecureValueCreate(sv *secretv0alpha1.SecureValue) field.ErrorList {
errs := make(field.ErrorList, 0)
if sv.Spec.Description == "" {
errs = append(errs, field.Required(field.NewPath("spec", "description"), "a `description` is required"))
}
if sv.Spec.Value == "" && (sv.Spec.Ref == nil || (sv.Spec.Ref != nil && *sv.Spec.Ref == "")) {
errs = append(errs, field.Required(field.NewPath("spec"), "either a `value` or `ref` is required"))
}
if sv.Spec.Value != "" && (sv.Spec.Ref != nil && *sv.Spec.Ref != "") {
errs = append(errs, field.Forbidden(field.NewPath("spec"), "only one of `value` or `ref` can be set"))
}
return errs
}
// validateSecureValueUpdate does basic spec validation of a securevalue for the Update operation.
func validateSecureValueUpdate(sv, oldSv *secretv0alpha1.SecureValue) field.ErrorList {
errs := make(field.ErrorList, 0)
// For updates, an `old` object is required.
if oldSv == nil {
errs = append(errs, field.InternalError(field.NewPath("spec"), errors.New("old object is nil")))
return errs
}
// Only validate if one of the fields is being changed/set.
if sv.Spec.Value != "" || (sv.Spec.Ref != nil && *sv.Spec.Ref != "") {
if (oldSv.Spec.Ref != nil && *oldSv.Spec.Ref != "") && sv.Spec.Value != "" {
errs = append(errs, field.Forbidden(field.NewPath("spec"), "cannot set `value` when `ref` was already previously set"))
}
if (oldSv.Spec.Ref == nil || (oldSv.Spec.Ref != nil && *oldSv.Spec.Ref == "")) && (sv.Spec.Ref != nil && *sv.Spec.Ref != "") {
errs = append(errs, field.Forbidden(field.NewPath("spec"), "cannot set `ref` when `value` was already previously set"))
}
}
// Keeper cannot be changed.
if sv.Spec.Keeper != oldSv.Spec.Keeper {
errs = append(errs, field.Forbidden(field.NewPath("spec"), "the `keeper` cannot be changed"))
}
return errs
}
// validateDecrypters validates that (if populated) the `decrypters` must be unique.
func validateDecrypters(decrypters []string, decryptersAllowList map[string]struct{}) field.ErrorList {
errs := make(field.ErrorList, 0)
// Limit the number of decrypters to 64 to not have it unbounded.
// The number was chosen arbitrarily and should be enough.
if len(decrypters) > 64 {
errs = append(
errs,
field.TooMany(field.NewPath("spec", "decrypters"), len(decrypters), 64),
)
return errs
}
decrypterNames := make(map[string]struct{}, 0)
for i, decrypter := range decrypters {
decrypter = strings.TrimSpace(decrypter)
if decrypter == "" {
errs = append(
errs,
field.Invalid(field.NewPath("spec", "decrypters", "["+strconv.Itoa(i)+"]"), decrypter, "decrypters cannot be empty if specified"),
)
continue
}
// Allow List: decrypters must match exactly and be in the allowed list to be able to decrypt.
if len(decryptersAllowList) > 0 {
if _, exists := decryptersAllowList[decrypter]; !exists {
errs = append(
errs,
field.Invalid(field.NewPath("spec", "decrypters", "["+strconv.Itoa(i)+"]"), decrypter, fmt.Sprintf("allowed values: %v", decryptersAllowList)),
)
return errs
}
continue
}
// Use the same validation as labels for the decrypters.
if verrs := validation.IsValidLabelValue(decrypter); len(verrs) > 0 {
for _, verr := range verrs {
errs = append(
errs,
field.Invalid(field.NewPath("spec", "decrypters", "["+strconv.Itoa(i)+"]"), decrypter, verr),
)
}
continue
}
if _, exists := decrypterNames[decrypter]; exists {
errs = append(
errs,
field.Invalid(field.NewPath("spec", "decrypters", "["+strconv.Itoa(i)+"]"), decrypter, "decrypters must be unique"),
)
continue
}
decrypterNames[decrypter] = struct{}{}
}
return errs
}
pkg/registry/apis/secret/reststorage/secure_value_rest_test.go
-308
View File
@@ -1,308 +0,0 @@
package reststorage
import (
"fmt"
"maps"
"slices"
"strings"
"testing"
"github.com/stretchr/testify/require"
"k8s.io/apiserver/pkg/admission"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
)
func TestValidateSecureValue(t *testing.T) {
t.Run("when creating a new securevalue", func(t *testing.T) {
keeper := "keeper"
validSecureValue := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description",
Value: "value",
Keeper: &keeper,
Decrypters: []string{"app1", "app2"},
},
}
t.Run("the `description` must be present", func(t *testing.T) {
sv := validSecureValue.DeepCopy()
sv.Spec.Description = ""
errs := ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec.description", errs[0].Field)
})
t.Run("either a `value` or `ref` must be present but not both", func(t *testing.T) {
sv := validSecureValue.DeepCopy()
sv.Spec.Value = ""
sv.Spec.Ref = nil
errs := ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
ref := "value"
sv.Spec.Value = "value"
sv.Spec.Ref = &ref
errs = ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("`value` cannot exceed 24576 bytes", func(t *testing.T) {
sv := validSecureValue.DeepCopy()
sv.Spec.Value = secretv0alpha1.NewExposedSecureValue(strings.Repeat("a", contracts.SECURE_VALUE_RAW_INPUT_MAX_SIZE_BYTES+1))
sv.Spec.Ref = nil
errs := ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec.value", errs[0].Field)
})
})
t.Run("when updating a securevalue", func(t *testing.T) {
t.Run("when trying to switch from a `value` (old) to a `ref` (new), it returns an error", func(t *testing.T) {
oldSv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Ref: nil, // empty `ref` means a `value` was present.
},
}
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Ref: &ref,
},
}
errs := ValidateSecureValue(sv, oldSv, admission.Update, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("when trying to switch from a `ref` (old) to a `value` (new), it returns an error", func(t *testing.T) {
ref := "non-empty"
oldSv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Ref: &ref,
},
}
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Value: "value",
},
}
errs := ValidateSecureValue(sv, oldSv, admission.Update, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("when both `value` and `ref` are set, it returns an error", func(t *testing.T) {
refNonEmpty := "non-empty"
oldSv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Ref: &refNonEmpty,
},
}
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Value: "value",
Ref: &ref,
},
}
errs := ValidateSecureValue(sv, oldSv, admission.Update, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
oldSv = &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Value: "non-empty",
},
}
errs = ValidateSecureValue(sv, oldSv, admission.Update, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("when no changes are made, it returns no errors", func(t *testing.T) {
oldSv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "old-description",
},
}
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "new-description",
},
}
errs := ValidateSecureValue(sv, oldSv, admission.Update, nil)
require.Empty(t, errs)
})
t.Run("when the old object is `nil` it returns an error", func(t *testing.T) {
sv := &secretv0alpha1.SecureValue{}
errs := ValidateSecureValue(sv, nil, admission.Update, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
t.Run("when trying to change the `keeper`, it returns an error", func(t *testing.T) {
keeperA := "a-keeper"
keeperAnother := "another-keeper"
oldSv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Keeper: &keeperA,
},
}
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Keeper: &keeperAnother,
},
}
errs := ValidateSecureValue(sv, oldSv, admission.Update, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec", errs[0].Field)
})
})
t.Run("`decrypters` must have unique items", func(t *testing.T) {
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: []string{
"app1",
"app1",
},
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec.decrypters.[1]", errs[0].Field)
})
t.Run("when set, the `decrypters` must be one of the allowed in the allow list", func(t *testing.T) {
allowList := map[string]struct{}{"app1": {}, "app2": {}}
decrypters := slices.Collect(maps.Keys(allowList))
t.Run("no matches, returns an error", func(t *testing.T) {
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: []string{"app3"},
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, allowList)
require.Len(t, errs, 1)
})
t.Run("no decrypters, returns no error", func(t *testing.T) {
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: []string{},
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, allowList)
require.Empty(t, errs)
})
t.Run("one match, returns no errors", func(t *testing.T) {
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: []string{decrypters[0]},
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, allowList)
require.Empty(t, errs)
})
t.Run("all matches, returns no errors", func(t *testing.T) {
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: decrypters,
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, allowList)
require.Empty(t, errs)
})
})
t.Run("`decrypters` must be a valid label value", func(t *testing.T) {
decrypters := []string{
"", // invalid
"is/this/valid", // invalid
"is this valid", // invalid
"is.this.valid",
"is-this-valid",
"is_this_valid",
"0isthisvalid9",
"isthisvalid9",
"0isthisvalid",
"isthisvalid",
}
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: decrypters,
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 3)
})
t.Run("`decrypters` cannot have more than 64 items", func(t *testing.T) {
decrypters := make([]string, 0, 64+1)
for i := 0; i < 64+1; i++ {
decrypters = append(decrypters, fmt.Sprintf("app%d", i))
}
ref := "ref"
sv := &secretv0alpha1.SecureValue{
Spec: secretv0alpha1.SecureValueSpec{
Description: "description", Ref: &ref,
Decrypters: decrypters,
},
}
errs := ValidateSecureValue(sv, nil, admission.Create, nil)
require.Len(t, errs, 1)
require.Equal(t, "spec.decrypters", errs[0].Field)
})
}
pkg/registry/apis/secret/secretkeeper/secretkeeper.go
+3 -2
View File
@@ -1,8 +1,9 @@
package secretkeeper
import (
"go.opentelemetry.io/otel/trace"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/sqlkeeper"
)
@@ -15,7 +16,7 @@ type OSSKeeperService struct {
var _ contracts.KeeperService = (*OSSKeeperService)(nil)
func ProvideService(
tracer tracing.Tracer,
tracer trace.Tracer,
store contracts.EncryptedValueStorage,
encryptionManager contracts.EncryptionManager,
) (*OSSKeeperService, error) {
pkg/registry/apis/secret/secretkeeper/secretkeeper_test.go
+4 -2
View File
@@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry/apis/secret/secretkeeper/sqlkeeper"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tests/testsuite"
@@ -31,8 +31,10 @@ func Test_OSSKeeperService_GetKeepers(t *testing.T) {
}
func setupTestService(t *testing.T, cfg *setting.Cfg) (*OSSKeeperService, error) {
tracer := noop.NewTracerProvider().Tracer("test")
// Initialize the keeper service
keeperService, err := ProvideService(tracing.InitializeTracerForTest(), nil, nil)
keeperService, err := ProvideService(tracer, nil, nil)
return keeperService, err
}
pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper.go
+22 -8
View File
@@ -5,12 +5,13 @@ import (
"fmt"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
type SQLKeeper struct {
tracer tracing.Tracer
tracer trace.Tracer
encryptionManager contracts.EncryptionManager
store contracts.EncryptedValueStorage
}
@@ -18,7 +19,7 @@ type SQLKeeper struct {
var _ contracts.Keeper = (*SQLKeeper)(nil)
func NewSQLKeeper(
tracer tracing.Tracer,
tracer trace.Tracer,
encryptionManager contracts.EncryptionManager,
store contracts.EncryptedValueStorage,
) *SQLKeeper {
@@ -31,7 +32,7 @@ func NewSQLKeeper(
// TODO: parameter cfg is not being used
func (s *SQLKeeper) Store(ctx context.Context, _ secretv0alpha1.KeeperConfig, namespace string, exposedValueOrRef string) (contracts.ExternalID, error) {
ctx, span := s.tracer.Start(ctx, "sqlKeeper.Store")
ctx, span := s.tracer.Start(ctx, "SQLKeeper.Store", trace.WithAttributes(attribute.String("namespace", namespace)))
defer span.End()
encryptedData, err := s.encryptionManager.Encrypt(ctx, namespace, []byte(exposedValueOrRef))
@@ -44,11 +45,18 @@ func (s *SQLKeeper) Store(ctx context.Context, _ secretv0alpha1.KeeperConfig, na
return "", fmt.Errorf("unable to store encrypted value: %w", err)
}
return contracts.ExternalID(encryptedVal.UID), nil
externalID := contracts.ExternalID(encryptedVal.UID)
span.SetAttributes(attribute.String("externalID", externalID.String()))
return externalID, nil
}
func (s *SQLKeeper) Expose(ctx context.Context, cfg secretv0alpha1.KeeperConfig, namespace string, externalID contracts.ExternalID) (secretv0alpha1.ExposedSecureValue, error) {
ctx, span := s.tracer.Start(ctx, "sqlKeeper.Expose")
ctx, span := s.tracer.Start(ctx, "SQLKeeper.Expose", trace.WithAttributes(
attribute.String("namespace", namespace),
attribute.String("externalID", externalID.String()),
))
defer span.End()
encryptedValue, err := s.store.Get(ctx, namespace, externalID.String())
@@ -66,7 +74,10 @@ func (s *SQLKeeper) Expose(ctx context.Context, cfg secretv0alpha1.KeeperConfig,
}
func (s *SQLKeeper) Delete(ctx context.Context, cfg secretv0alpha1.KeeperConfig, namespace string, externalID contracts.ExternalID) error {
ctx, span := s.tracer.Start(ctx, "sqlKeeper.Delete")
ctx, span := s.tracer.Start(ctx, "SQLKeeper.Delete", trace.WithAttributes(
attribute.String("namespace", namespace),
attribute.String("externalID", externalID.String()),
))
defer span.End()
err := s.store.Delete(ctx, namespace, externalID.String())
@@ -77,7 +88,10 @@ func (s *SQLKeeper) Delete(ctx context.Context, cfg secretv0alpha1.KeeperConfig,
}
func (s *SQLKeeper) Update(ctx context.Context, cfg secretv0alpha1.KeeperConfig, namespace string, externalID contracts.ExternalID, exposedValueOrRef string) error {
ctx, span := s.tracer.Start(ctx, "sqlKeeper.Update")
ctx, span := s.tracer.Start(ctx, "SQLKeeper.Update", trace.WithAttributes(
attribute.String("namespace", namespace),
attribute.String("externalID", externalID.String()),
))
defer span.End()
encryptedData, err := s.encryptionManager.Encrypt(ctx, namespace, []byte(exposedValueOrRef))
pkg/registry/apis/secret/secretkeeper/sqlkeeper/keeper_test.go
+4 -2
View File
@@ -9,8 +9,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/setting"
)
@@ -136,6 +136,8 @@ func Test_SQLKeeperSetup(t *testing.T) {
}
func setupTestService(t *testing.T, cfg *setting.Cfg) (*SQLKeeper, error) {
tracer := noop.NewTracerProvider().Tracer("test")
// Initialize the encryption manager with in-memory implementation
encMgr := &inMemoryEncryptionManager{}
@@ -143,7 +145,7 @@ func setupTestService(t *testing.T, cfg *setting.Cfg) (*SQLKeeper, error) {
encValueStore := newInMemoryEncryptedValueStorage()
// Initialize the SQLKeeper
sqlKeeper := NewSQLKeeper(tracing.InitializeTracerForTest(), encMgr, encValueStore)
sqlKeeper := NewSQLKeeper(tracer, encMgr, encValueStore)
return sqlKeeper, nil
}
pkg/registry/apis/wireset.go
+3 -1
View File
@@ -46,6 +46,9 @@ var WireSet = wire.NewSet(
wire.Bind(new(datasource.PluginContextWrapper), new(*plugincontext.Provider)),
datasource.ProvideDefaultPluginConfigs,
// Secrets
secret.RegisterDependencies,
// Each must be added here *and* in the ServiceSink above
dashboardinternal.RegisterAPIService,
dashboardsnapshot.RegisterAPIService,
@@ -58,7 +61,6 @@ var WireSet = wire.NewSet(
provisioning.RegisterAPIService,
service.RegisterAPIService,
query.RegisterAPIService,
secret.RegisterAPIService,
userstorage.RegisterAPIService,
ofrep.RegisterAPIService,
)
pkg/server/wire_gen.go
+12 -30
View File
@@ -732,20 +732,6 @@ func Initialize(cfg *setting.Cfg, opts Options, apiOpts api.ServerOptions) (*Ser
return nil, err
}
userStorageAPIBuilder := userstorage.RegisterAPIService(featureToggles, apiserverService, registerer)
databaseDatabase := database5.ProvideDatabase(sqlStore)
secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(databaseDatabase, featureToggles)
if err != nil {
return nil, err
}
keeperMetadataStorage, err := metadata.ProvideKeeperMetadataStorage(databaseDatabase, featureToggles)
if err != nil {
return nil, err
}
secretDBMigrator := migrator2.NewWithEngine(sqlStore)
secretAPIBuilder, err := secret.RegisterAPIService(featureToggles, cfg, apiserverService, tracingService, secureValueMetadataStorage, keeperMetadataStorage, accessClient, acimplService, secretDBMigrator)
if err != nil {
return nil, err
}
factory := github.ProvideFactory()
legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, accessControl)
webhookExtraBuilder := webhooks.ProvideWebhooks(cfg, featureToggles, secretsService, factory, renderingService, resourceClient, eventualRestConfigProvider)
@@ -759,7 +745,12 @@ func Initialize(cfg *setting.Cfg, opts Options, apiOpts api.ServerOptions) (*Ser
return nil, err
}
ofrepAPIBuilder := ofrep.RegisterAPIService(apiserverService, cfg, staticFlagEvaluator)
apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, userStorageAPIBuilder, secretAPIBuilder, apiBuilder, ofrepAPIBuilder)
secretDBMigrator := migrator2.NewWithEngine(sqlStore)
dependencyRegisterer, err := secret.RegisterDependencies(featureToggles, cfg, secretDBMigrator, acimplService)
if err != nil {
return nil, err
}
apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, userStorageAPIBuilder, apiBuilder, ofrepAPIBuilder, dependencyRegisterer)
teamPermissionsService, err := ossaccesscontrol.ProvideTeamPermissions(cfg, featureToggles, routeRegisterImpl, sqlStore, accessControl, ossLicensingService, acimplService, teamService, userService, actionSetService)
if err != nil {
return nil, err
@@ -1254,20 +1245,6 @@ func InitializeForTest(t sqlutil.ITestDB, testingT interface {
return nil, err
}
userStorageAPIBuilder := userstorage.RegisterAPIService(featureToggles, apiserverService, registerer)
databaseDatabase := database5.ProvideDatabase(sqlStore)
secureValueMetadataStorage, err := metadata.ProvideSecureValueMetadataStorage(databaseDatabase, featureToggles)
if err != nil {
return nil, err
}
keeperMetadataStorage, err := metadata.ProvideKeeperMetadataStorage(databaseDatabase, featureToggles)
if err != nil {
return nil, err
}
secretDBMigrator := migrator2.NewWithEngine(sqlStore)
secretAPIBuilder, err := secret.RegisterAPIService(featureToggles, cfg, apiserverService, tracingService, secureValueMetadataStorage, keeperMetadataStorage, accessClient, acimplService, secretDBMigrator)
if err != nil {
return nil, err
}
factory := github.ProvideFactory()
legacyMigrator := legacy.ProvideLegacyMigrator(sqlStore, provisioningServiceImpl, libraryPanelService, accessControl)
webhookExtraBuilder := webhooks.ProvideWebhooks(cfg, featureToggles, secretsService, factory, renderingService, resourceClient, eventualRestConfigProvider)
@@ -1281,7 +1258,12 @@ func InitializeForTest(t sqlutil.ITestDB, testingT interface {
return nil, err
}
ofrepAPIBuilder := ofrep.RegisterAPIService(apiserverService, cfg, staticFlagEvaluator)
apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, userStorageAPIBuilder, secretAPIBuilder, apiBuilder, ofrepAPIBuilder)
secretDBMigrator := migrator2.NewWithEngine(sqlStore)
dependencyRegisterer, err := secret.RegisterDependencies(featureToggles, cfg, secretDBMigrator, acimplService)
if err != nil {
return nil, err
}
apiregistryService := apiregistry.ProvideRegistryServiceSink(dashboardsAPIBuilder, snapshotsAPIBuilder, featureFlagAPIBuilder, dataSourceAPIBuilder, folderAPIBuilder, identityAccessManagementAPIBuilder, queryAPIBuilder, userStorageAPIBuilder, apiBuilder, ofrepAPIBuilder, dependencyRegisterer)
teamPermissionsService, err := ossaccesscontrol.ProvideTeamPermissions(cfg, featureToggles, routeRegisterImpl, sqlStore, accessControl, ossLicensingService, acimplService, teamService, userService, actionSetService)
if err != nil {
return nil, err
pkg/services/authz/zanzana/README.md
+251 -5
View File
@@ -1,6 +1,251 @@
# Zanzana
Zanzana is authorization server and wrapper around OpenFGA. OpenFGA implements Zanzibar authorization model, which is relation-based access control. But it's pretty flexible, so you can use it for implementing various authorization models.
## Running Zanzana in embedded mode
By default Zanzana runs in the same binary as Grafana, it's called embedded mode. Grafana communicates with Zanzana/OpenFGA via in-proc GRPC. OpenFGA supports several DB types, like MySQL, Postgres, sqlite. Default is sqlite, but since we run Postgres in cloud, this is recommended setup. In case of Postgres and MySQL OpenFGA creates tables in the same database as grafana (default db name is "grafana"). Minimal config for running Zanzana with Postgres is this:
```ini
app_mode = development
[log]
level = info
[feature_toggles]
zanzana = true
[database]
type = postgres
host = 127.0.0.1:5432
name = grafana
user = grafana
password = password
```
To run postgres DB you need to create docker compose file. Switch to the `devenv` directory and run script to create it:
```sh
cd devenv
./create_docker_compose.sh postgres
docker compose up -d
```
or simply `make devenv sources=postgres` from repo root.
Now you can run grafana (from source root directory):
```sh
make run
```
## Instrumentation
It's always good to know what happens inside and have tools to check performance and request path. So it's good idea to instrument grafana instance with metrics and traces. You can do it by running prometheus and tempo in docker and configure grafana to send traces.
To run prometheus and tempo, add it to docker compose file:
```sh
./create_docker_compose.sh postgres, tempo, grafana
```
There're some differences in compose files and some unnecessary blocks, so you can simply copy this docker-compose file:
```yaml
services:
postgres:
image: postgres:15.7
environment:
POSTGRES_USER: grafana
POSTGRES_PASSWORD: password
POSTGRES_DB: grafana
ports:
- "5432:5432"
command: postgres -c log_connections=on -c log_disconnections=on -c log_destination=stderr
healthcheck:
test: [ "CMD", "pg_isready", "-q", "-d", "grafana", "-U", "grafana" ]
timeout: 45s
interval: 10s
retries: 10
grafana:
image: grafana/grafana:latest
ports:
- "3001:3000"
volumes:
- "./dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml"
- "./datasources_docker.yaml:/etc/grafana/provisioning/datasources/datasources.yaml"
environment:
GF_RENDERING_SERVER_URL: http://renderer:8081/render
GF_RENDERING_CALLBACK_URL: http://grafana:3000/
prometheus:
image: prom/prometheus:latest
command:
- --config.file=/etc/prometheus.yaml
- --web.enable-remote-write-receiver
- --enable-feature=exemplar-storage
volumes:
- ./docker/blocks/prometheus/prometheus.yml:/etc/prometheus.yaml
ports:
- "9090:9090"
labels:
namespace: monitoring
logging:
driver: loki
options:
loki-url: 'http://localhost:3100/api/prom/push'
labels: namespace
tempo:
image: grafana/tempo:latest
command:
- --config.file=/etc/tempo.yaml
volumes:
- ./docker/blocks/tempo/tempo.yaml:/etc/tempo.yaml
- ./docker/blocks/tempo/tempo-data:/tmp/tempo
ports:
- "14268:14268" # jaeger ingest
- "3200:3200" # tempo
- "4317:4317" # otlp grpc
- "4318:4318" # otlp http
```
```sh
docker-compose up -d
```
Then you need to configure grafana to send telemetry to Tempo:
```ini
[server]
router_logging = true
[tracing.opentelemetry.otlp]
address = localhost:4317
```
Then start grafana.
In order to use traces, you'll need to create Tempo data source in Grafana. Go to the grafana instance running in docker (http://localhost:3001) and create new Tempo data source with URL `http://tempo:3200`. Now you can use `gdev-prometheus` and `tempo` data sources for building dashboards or looking into metrics/traces in Explore.
## Load testing
For load testing you can use [grafana-api-tests](https://github.com/grafana/grafana-api-tests) repo. Clone it and navigate to `simulation/fake-user-generator` folder. Run `./generateNestedFolders.ts` script to populate data in grafana:
```sh
./generateNestedFolders.ts --scenario medium -v --user admin --password <your_grafana_password>
```
For the load testing run [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) and specify desired test:
```sh
GRAFANA_PASSWORD=<password> k6 run loadtest/tests/dashboard_by_uid.js
```
## Running Zanzana standalone server
When running as a standalone server, Grafana communicates with Zanzana via GRPC. It also requires some additional GRPC authentication configuration. So you need to add auth-signer service to your docker compose file:
```yaml
auth-signer:
build: ./docker/blocks/auth/signer/.
ports:
- "6481:8080"
volumes:
- ./docker/blocks/auth/signer/config.yaml:/app/config.yaml
restart: unless-stopped
```
This service is located in the grafana-enterprise repo, so make sure you have linked enterprise repo:
```sh
# From OSS repo
make enterprise-to-oss
docker compose up -d
```
Then add grafana configured in client mode to be able to test requests against zanzana server. Unfortunately, due to issue with config implementation, stack id cannot be configured through env variables, so you should create ini file and put it there:
```ini
[environment]
stack_id = 11
```
Then link it to the grafana client. Default stack id is `11` and token is `ThisIsMySecretToken`. Those values configured in `auth-signer` by default, so if you need to change it, follow instructions in [auth signer readme](https://github.com/grafana/grafana-enterprise/blob/main/src/devenv/blocks/auth/signer/README.md)
```yaml
grafana-client:
image: grafana/grafana:main
ports:
- "3002:3000"
volumes:
- "./dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml"
- "./dev-dashboards:/usr/share/grafana/devenv/dev-dashboards"
- "./datasources_docker.yaml:/etc/grafana/provisioning/datasources/datasources.yaml"
- "<path_to_grafana_config_file_ini>:/etc/grafana/grafana.ini"
environment:
GF_DEFAULT_APP_MODE: development
GF_LOG_LEVEL: debug
GF_ENVIRONMENT_STACK_ID: 11
GF_FEATURE_TOGGLES_ENABLE: zanzana authZGRPCServer unifiedStorage unifiedStorageSearch
GF_ZANZANA_CLIENT_MODE: client
GF_ZANZANA_CLIENT_ADDRESS: host.docker.internal:10000
GF_ZANZANA_CLIENT_TOKEN: ThisIsMySecretToken
GF_ZANZANA_CLIENT_TOKEN_EXCHANGE_URL: http://host.docker.internal:6481/sign/access-token
```
Run containers:
```sh
docker compose up -d
```
Finally, configure zanzana standalone server (this is your `custom.ini` file in grafana repo):
```ini
app_mode = development
target = zanzana-server
[log]
level = debug
[feature_toggles]
zanzana = true
[zanzana.server]
check_query_cache = true
signing_keys_url = http://localhost:6481/jwks
[grpc_server]
enabled = true
address = 127.0.0.1:10000
[database]
type = postgres
host = 127.0.0.1:5432
name = grafana
user = grafana
password = password
[tracing.opentelemetry.otlp]
address = localhost:4317
```
Now you can run zanzana server:
```sh
make build-go
./bin/<arch>/grafana server target
```
To test everything out, go to grafana client (http://localhost:3002) and open any dashboard. There should be some records in the zanzana server logs
If you want to debug zanzana server, you can run it from VS Code debug panel - select `Run Authz server` target. Make sure that no grafana instances running at port `3001` since it's default port for this debug config (`docker compose stop grafana` if you have grafana running in docker, or remap it to another port).
## Zanzana cli
Zanzana can be run as a standalone OpenFGA HTTP server that allows you to use the OpenFGA CLI to debug and manage fine-grained authorization relationships within Grafana.
To test this you need to run standalone zanzana server. Use following config:
@@ -14,6 +259,7 @@ target = zanzana-server
zanzana = true
[zanzana.server]
allow_insecure = true
http_addr = 127.0.0.1:8080
[grpc_server]
@@ -24,19 +270,19 @@ address = 127.0.0.1:10000
And then run grafana server target:
```bash
./bin/darwin-arm64/grafana server target
./bin/<arch>/grafana server target
```
### Using OpenFGA CLI
useful info on how to setup and use https://openfga.dev/docs/getting-started/cli
Once the server is running, you can interact with it using the OpenFGA CLI:
There's useful info on how to setup and use [OpenFGA CLI](https://openfga.dev/docs/getting-started/cli). Once the server is running, you can interact with it using the CLI:
```bash
# List all stores
fga store list
# Other commands
fga model read
fga tuple list
fga model list --store-id <store_id>
fga tuple read --store-id <store_id>
fga query check
```
pkg/services/featuremgmt/registry.go
+5 -8
View File
@@ -698,14 +698,11 @@ var (
Owner: grafanaDatavizSquad,
},
{
// this is mainly used as a way to quickly disable query hints as a safeguard for our infrastructure
Name: "lokiQueryHints",
Description: "Enables query hints for Loki",
Stage: FeatureStageGeneralAvailability,
FrontendOnly: true,
Expression: "true",
Owner: grafanaObservabilityLogsSquad,
AllowSelfServe: false,
Name: "regressionTransformation",
Description: "Enables regression analysis transformation",
Stage: FeatureStagePublicPreview,
FrontendOnly: true,
Owner: grafanaDatavizSquad,
},
{
Name: "kubernetesFeatureToggles",
pkg/services/featuremgmt/toggles_gen.csv
+1 -1
View File
@@ -92,7 +92,7 @@ logsInfiniteScrolling,GA,@grafana/observability-logs,false,false,true
logRowsPopoverMenu,GA,@grafana/observability-logs,false,false,true
pluginsSkipHostEnvVars,experimental,@grafana/plugins-platform-backend,false,false,false
tableSharedCrosshair,experimental,@grafana/dataviz-squad,false,false,true
lokiQueryHints,GA,@grafana/observability-logs,false,false,true
regressionTransformation,preview,@grafana/dataviz-squad,false,false,true
kubernetesFeatureToggles,experimental,@grafana/grafana-operator-experience-squad,false,false,true
cloudRBACRoles,preview,@grafana/identity-access-team,false,true,false
alertingQueryOptimization,GA,@grafana/alerting-squad,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
92 logRowsPopoverMenu GA @grafana/observability-logs false false true
93 pluginsSkipHostEnvVars experimental @grafana/plugins-platform-backend false false false
94 tableSharedCrosshair experimental @grafana/dataviz-squad false false true
95 lokiQueryHints regressionTransformation GA preview @grafana/observability-logs @grafana/dataviz-squad false false true
96 kubernetesFeatureToggles experimental @grafana/grafana-operator-experience-squad false false true
97 cloudRBACRoles preview @grafana/identity-access-team false true false
98 alertingQueryOptimization GA @grafana/alerting-squad false false false
pkg/services/featuremgmt/toggles_gen.go
+3 -3
View File
@@ -379,9 +379,9 @@ const (
// Enables shared crosshair in table panel
FlagTableSharedCrosshair = "tableSharedCrosshair"
// FlagLokiQueryHints
// Enables query hints for Loki
FlagLokiQueryHints = "lokiQueryHints"
// FlagRegressionTransformation
// Enables regression analysis transformation
FlagRegressionTransformation = "regressionTransformation"
// FlagKubernetesFeatureToggles
// Use the kubernetes API for feature toggle management in the frontend
pkg/services/featuremgmt/toggles_gen.json
+3 -2
View File
@@ -1868,8 +1868,9 @@
{
"metadata": {
"name": "lokiQueryHints",
"resourceVersion": "1750434297879",
"creationTimestamp": "2023-12-18T20:43:16Z"
"resourceVersion": "1743693517832",
"creationTimestamp": "2023-12-18T20:43:16Z",
"deletionTimestamp": "2025-06-12T12:14:47Z"
},
"spec": {
"description": "Enables query hints for Loki",
pkg/storage/secret/database/database.go
+30 -14
View File
@@ -6,6 +6,8 @@ import (
"errors"
"github.com/jmoiron/sqlx"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
@@ -19,6 +21,7 @@ type contextSessionTxKey struct{}
type Database struct {
dbType string
sqlx *sqlx.DB
tracer trace.Tracer
// Keep the xorm.Engine instance and its references alive until the apiserver is shut down.
// This is only needed because the xorm.Engine calls a runtime.SetFinalizer, in a RAII-like pattern to close the DB,
@@ -34,13 +37,14 @@ type Database struct {
engine *xorm.Engine
}
func ProvideDatabase(db db.DB) *Database {
func ProvideDatabase(db db.DB, tracer trace.Tracer) *Database {
engine := db.GetEngine()
return &Database{
dbType: string(db.GetDBType()),
sqlx: sqlx.NewDb(engine.DB().DB, db.GetDialect().DriverName()),
engine: engine,
tracer: tracer,
}
}
@@ -48,25 +52,31 @@ func (db *Database) DriverName() string {
return db.dbType
}
func (db *Database) Transaction(ctx context.Context, callback func(context.Context) error) error {
txCtx := ctx
func (db *Database) Transaction(ctx context.Context, callback func(context.Context) error) (err error) {
// If another transaction is already open, we just use that one instead of nesting.
sqlxTx, ok := txCtx.Value(contextSessionTxKey{}).(*sqlx.Tx)
sqlxTx, ok := ctx.Value(contextSessionTxKey{}).(*sqlx.Tx)
if sqlxTx != nil && ok {
// We are already in a transaction, so we don't commit or rollback, let the outermost transaction do it.
return callback(txCtx)
return callback(ctx)
}
tx, err := db.sqlx.Beginx()
spanCtx, span := db.tracer.Start(ctx, "Database.Transaction")
defer span.End()
defer func() {
if err != nil {
span.SetStatus(codes.Error, "Transaction failed")
span.RecordError(err)
}
}()
sqlxTx, err = db.sqlx.BeginTxx(spanCtx, nil)
if err != nil {
return err
}
sqlxTx = tx
// Save it in the context so the transaction can be reused in case it is nested.
txCtx = context.WithValue(ctx, contextSessionTxKey{}, sqlxTx)
txCtx := context.WithValue(spanCtx, contextSessionTxKey{}, sqlxTx)
if err := callback(txCtx); err != nil {
if rbErr := sqlxTx.Rollback(); rbErr != nil {
@@ -80,19 +90,25 @@ func (db *Database) Transaction(ctx context.Context, callback func(context.Conte
}
func (db *Database) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
spanCtx, span := db.tracer.Start(ctx, "Database.ExecContext")
defer span.End()
// If another transaction is already open, we just use that one instead of nesting.
if tx, ok := ctx.Value(contextSessionTxKey{}).(*sqlx.Tx); tx != nil && ok {
return tx.ExecContext(ctx, db.sqlx.Rebind(query), args...)
return tx.ExecContext(spanCtx, db.sqlx.Rebind(query), args...)
}
return db.sqlx.ExecContext(ctx, db.sqlx.Rebind(query), args...)
return db.sqlx.ExecContext(spanCtx, db.sqlx.Rebind(query), args...)
}
func (db *Database) QueryContext(ctx context.Context, query string, args ...any) (contracts.Rows, error) {
spanCtx, span := db.tracer.Start(ctx, "Database.QueryContext")
defer span.End()
// If another transaction is already open, we just use that one instead of nesting.
if tx, ok := ctx.Value(contextSessionTxKey{}).(*sqlx.Tx); tx != nil && ok {
return tx.QueryContext(ctx, db.sqlx.Rebind(query), args...)
return tx.QueryContext(spanCtx, db.sqlx.Rebind(query), args...)
}
return db.sqlx.QueryContext(ctx, db.sqlx.Rebind(query), args...)
return db.sqlx.QueryContext(spanCtx, db.sqlx.Rebind(query), args...)
}
pkg/storage/secret/encryption/data_key_store.go
+1 -7
View File
@@ -9,7 +9,6 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
@@ -22,12 +21,7 @@ type encryptionStoreImpl struct {
log log.Logger
}
func ProvideDataKeyStorage(
db contracts.Database,
tracer trace.Tracer,
features featuremgmt.FeatureToggles,
registerer prometheus.Registerer,
) (contracts.DataKeyStorage, error) {
func ProvideDataKeyStorage(db contracts.Database, tracer trace.Tracer, features featuremgmt.FeatureToggles) (contracts.DataKeyStorage, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) ||
!features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
return &encryptionStoreImpl{}, nil
pkg/storage/secret/encryption/data_key_store_test.go
+1 -1
View File
@@ -31,7 +31,7 @@ func TestEncryptionStoreImpl_DataKeyLifecycle(t *testing.T) {
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
tracer := noop.NewTracerProvider().Tracer("test")
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
store, err := ProvideDataKeyStorage(database.ProvideDatabase(testDB), tracer, features, nil)
store, err := ProvideDataKeyStorage(database.ProvideDatabase(testDB, tracer), tracer, features)
require.NoError(t, err)
ctx := context.Background()
pkg/storage/secret/encryption/encrypted_value_store.go
+35 -2
View File
@@ -7,6 +7,8 @@ import (
"time"
"github.com/google/uuid"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -17,7 +19,7 @@ var (
ErrEncryptedValueNotFound = errors.New("encrypted value not found")
)
func ProvideEncryptedValueStorage(db contracts.Database, features featuremgmt.FeatureToggles) (contracts.EncryptedValueStorage, error) {
func ProvideEncryptedValueStorage(db contracts.Database, tracer trace.Tracer, features featuremgmt.FeatureToggles) (contracts.EncryptedValueStorage, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) ||
!features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
return &encryptedValStorage{}, nil
@@ -26,15 +28,28 @@ func ProvideEncryptedValueStorage(db contracts.Database, features featuremgmt.Fe
return &encryptedValStorage{
db: db,
dialect: sqltemplate.DialectForDriver(db.DriverName()),
tracer: tracer,
}, nil
}
type encryptedValStorage struct {
db contracts.Database
dialect sqltemplate.Dialect
tracer trace.Tracer
}
func (s *encryptedValStorage) Create(ctx context.Context, namespace string, encryptedData []byte) (*contracts.EncryptedValue, error) {
func (s *encryptedValStorage) Create(ctx context.Context, namespace string, encryptedData []byte) (ev *contracts.EncryptedValue, err error) {
ctx, span := s.tracer.Start(ctx, "EncryptedValueStorage.Create", trace.WithAttributes(
attribute.String("namespace", namespace),
))
defer span.End()
defer func() {
if ev != nil {
span.SetAttributes(attribute.String("uid", ev.UID))
}
}()
createdTime := time.Now().Unix()
encryptedValue := &EncryptedValue{
UID: uuid.New().String(),
@@ -74,6 +89,12 @@ func (s *encryptedValStorage) Create(ctx context.Context, namespace string, encr
}
func (s *encryptedValStorage) Update(ctx context.Context, namespace string, uid string, encryptedData []byte) error {
ctx, span := s.tracer.Start(ctx, "EncryptedValueStorage.Update", trace.WithAttributes(
attribute.String("uid", uid),
attribute.String("namespace", namespace),
))
defer span.End()
req := updateEncryptedValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace,
@@ -102,6 +123,12 @@ func (s *encryptedValStorage) Update(ctx context.Context, namespace string, uid
}
func (s *encryptedValStorage) Get(ctx context.Context, namespace string, uid string) (*contracts.EncryptedValue, error) {
ctx, span := s.tracer.Start(ctx, "EncryptedValueStorage.Get", trace.WithAttributes(
attribute.String("uid", uid),
attribute.String("namespace", namespace),
))
defer span.End()
req := &readEncryptedValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace,
@@ -141,6 +168,12 @@ func (s *encryptedValStorage) Get(ctx context.Context, namespace string, uid str
}
func (s *encryptedValStorage) Delete(ctx context.Context, namespace string, uid string) error {
ctx, span := s.tracer.Start(ctx, "EncryptedValueStorage.Delete", trace.WithAttributes(
attribute.String("uid", uid),
attribute.String("namespace", namespace),
))
defer span.End()
req := deleteEncryptedValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace,
pkg/storage/secret/encryption/encrypted_value_store_test.go
+4 -2
View File
@@ -9,16 +9,18 @@ import (
"github.com/grafana/grafana/pkg/storage/secret/database"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
)
func TestEncryptedValueStoreImpl(t *testing.T) {
// Initialize data key storage with a fake db
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
database := database.ProvideDatabase(testDB)
tracer := noop.NewTracerProvider().Tracer("test")
database := database.ProvideDatabase(testDB, tracer)
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
ctx := context.Background()
store, err := ProvideEncryptedValueStorage(database, features)
store, err := ProvideEncryptedValueStorage(database, tracer, features)
require.NoError(t, err)
t.Run("creating an encrypted value returns it", func(t *testing.T) {
pkg/storage/secret/metadata/keeper_store.go
+65 -3
View File
@@ -9,17 +9,21 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/secret/xkube"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// keeperMetadataStorage is the actual implementation of the keeper metadata storage.
type keeperMetadataStorage struct {
db contracts.Database
dialect sqltemplate.Dialect
tracer trace.Tracer
}
var _ contracts.KeeperMetadataStorage = (*keeperMetadataStorage)(nil)
func ProvideKeeperMetadataStorage(db contracts.Database, features featuremgmt.FeatureToggles) (contracts.KeeperMetadataStorage, error) {
func ProvideKeeperMetadataStorage(db contracts.Database, tracer trace.Tracer, features featuremgmt.FeatureToggles) (contracts.KeeperMetadataStorage, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) ||
!features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
return &keeperMetadataStorage{}, nil
@@ -28,10 +32,18 @@ func ProvideKeeperMetadataStorage(db contracts.Database, features featuremgmt.Fe
return &keeperMetadataStorage{
db: db,
dialect: sqltemplate.DialectForDriver(db.DriverName()),
tracer: tracer,
}, nil
}
func (s *keeperMetadataStorage) Create(ctx context.Context, keeper *secretv0alpha1.Keeper, actorUID string) (*secretv0alpha1.Keeper, error) {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.Create", trace.WithAttributes(
attribute.String("name", keeper.GetName()),
attribute.String("namespace", keeper.GetNamespace()),
attribute.String("actorUID", actorUID),
))
defer span.End()
row, err := toKeeperCreateRow(keeper, actorUID)
if err != nil {
return nil, fmt.Errorf("failed to create row: %w", err)
@@ -82,6 +94,13 @@ func (s *keeperMetadataStorage) Create(ctx context.Context, keeper *secretv0alph
}
func (s *keeperMetadataStorage) Read(ctx context.Context, namespace xkube.Namespace, name string, opts contracts.ReadOpts) (*secretv0alpha1.Keeper, error) {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.Read", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
attribute.Bool("isForUpdate", opts.ForUpdate),
))
defer span.End()
keeperDB, err := s.read(ctx, namespace.String(), name, opts)
if err != nil {
return nil, err
@@ -134,6 +153,13 @@ func (s *keeperMetadataStorage) read(ctx context.Context, namespace, name string
}
func (s *keeperMetadataStorage) Update(ctx context.Context, newKeeper *secretv0alpha1.Keeper, actorUID string) (*secretv0alpha1.Keeper, error) {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.Update", trace.WithAttributes(
attribute.String("name", newKeeper.GetName()),
attribute.String("namespace", newKeeper.GetNamespace()),
attribute.String("actorUID", actorUID),
))
defer span.End()
var newRow *keeperDB
err := s.db.Transaction(ctx, func(ctx context.Context) error {
@@ -195,6 +221,12 @@ func (s *keeperMetadataStorage) Update(ctx context.Context, newKeeper *secretv0a
}
func (s *keeperMetadataStorage) Delete(ctx context.Context, namespace xkube.Namespace, name string) error {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.Delete", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
))
defer span.End()
req := deleteKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
@@ -224,7 +256,16 @@ func (s *keeperMetadataStorage) Delete(ctx context.Context, namespace xkube.Name
return nil
}
func (s *keeperMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) ([]secretv0alpha1.Keeper, error) {
func (s *keeperMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) (keeperList []secretv0alpha1.Keeper, err error) {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.List", trace.WithAttributes(
attribute.String("namespace", namespace.String()),
))
defer span.End()
defer func() {
span.SetAttributes(attribute.Int("returnedList.count", len(keeperList)))
}()
req := listKeeper{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
@@ -270,7 +311,20 @@ func (s *keeperMetadataStorage) List(ctx context.Context, namespace xkube.Namesp
// validateSecureValueReferences checks that all secure values referenced by the keeper exist and are not referenced by other third-party keepers.
// It is used by other methods inside a transaction.
func (s *keeperMetadataStorage) validateSecureValueReferences(ctx context.Context, keeper *secretv0alpha1.Keeper) error {
func (s *keeperMetadataStorage) validateSecureValueReferences(ctx context.Context, keeper *secretv0alpha1.Keeper) (err error) {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.ValidateSecureValueReferences", trace.WithAttributes(
attribute.String("name", keeper.GetName()),
attribute.String("namespace", keeper.GetNamespace()),
))
defer span.End()
defer func() {
if err != nil {
span.SetStatus(codes.Error, "failed to validate secure value references")
span.RecordError(err)
}
}()
usedSecureValues := extractSecureValues(keeper)
// No secure values are referenced, return early.
@@ -405,11 +459,19 @@ func (s *keeperMetadataStorage) validateSecureValueReferences(ctx context.Contex
}
func (s *keeperMetadataStorage) GetKeeperConfig(ctx context.Context, namespace string, name *string, opts contracts.ReadOpts) (secretv0alpha1.KeeperConfig, error) {
ctx, span := s.tracer.Start(ctx, "KeeperMetadataStorage.GetKeeperConfig", trace.WithAttributes(
attribute.String("namespace", namespace),
attribute.Bool("isForUpdate", opts.ForUpdate),
))
defer span.End()
// Check if keeper is the systemwide one.
if name == nil {
return nil, nil
}
span.SetAttributes(attribute.String("name", *name))
// Load keeper config from metadata store, or TODO: keeper cache.
kp, err := s.read(ctx, namespace, *name, opts)
if err != nil {
pkg/storage/secret/metadata/keeper_store_test.go
+4 -2
View File
@@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/storage/secret/database"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
)
func Test_KeeperMetadataStorage_GetKeeperConfig(t *testing.T) {
@@ -334,11 +335,12 @@ func Test_KeeperMetadataStorage_GetKeeperConfig(t *testing.T) {
func initStorage(t *testing.T) contracts.KeeperMetadataStorage {
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
db := database.ProvideDatabase(testDB)
tracer := noop.NewTracerProvider().Tracer("test")
db := database.ProvideDatabase(testDB, tracer)
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
// Initialize the keeper storage
keeperMetadataStorage, err := ProvideKeeperMetadataStorage(db, features)
keeperMetadataStorage, err := ProvideKeeperMetadataStorage(db, tracer, features)
require.NoError(t, err)
return keeperMetadataStorage
}
pkg/storage/secret/metadata/outbox_store.go
+40 -4
View File
@@ -7,6 +7,9 @@ import (
"time"
unifiedsql "github.com/grafana/grafana/pkg/storage/unified/sql"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/google/uuid"
"github.com/grafana/grafana/pkg/registry/apis/secret/assert"
@@ -17,12 +20,14 @@ import (
type outboxStore struct {
db contracts.Database
dialect sqltemplate.Dialect
tracer trace.Tracer
}
func ProvideOutboxQueue(db contracts.Database) contracts.OutboxQueue {
func ProvideOutboxQueue(db contracts.Database, tracer trace.Tracer) contracts.OutboxQueue {
return &outboxStore{
db: db,
dialect: sqltemplate.DialectForDriver(db.DriverName()),
tracer: tracer,
}
}
@@ -39,10 +44,29 @@ type outboxMessageDB struct {
Created int64
}
func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMessage) (string, error) {
func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMessage) (messageID string, err error) {
ctx, span := s.tracer.Start(ctx, "outboxStore.Append", trace.WithAttributes(
attribute.String("name", input.Name),
attribute.String("namespace", input.Namespace),
attribute.String("type", string(input.Type)),
attribute.String("requestID", input.RequestID),
))
defer span.End()
defer func() {
if err != nil {
span.SetStatus(codes.Error, "failed to append outbox message")
span.RecordError(err)
}
if messageID != "" {
span.SetAttributes(attribute.String("messageID", messageID))
}
}()
assert.True(input.Type != "", "outboxStore.Append: outbox message type is required")
messageID, err := s.insertMessage(ctx, input)
messageID, err = s.insertMessage(ctx, input)
if err != nil {
return messageID, fmt.Errorf("inserting message into outbox table: %+w", err)
}
@@ -189,7 +213,19 @@ func (s *outboxStore) ReceiveN(ctx context.Context, n uint) ([]contracts.OutboxM
return messages, nil
}
func (s *outboxStore) Delete(ctx context.Context, messageID string) error {
func (s *outboxStore) Delete(ctx context.Context, messageID string) (err error) {
ctx, span := s.tracer.Start(ctx, "outboxStore.Append", trace.WithAttributes(
attribute.String("messageID", messageID),
))
defer span.End()
defer func() {
if err != nil {
span.SetStatus(codes.Error, "failed to delete message from outbox")
span.RecordError(err)
}
}()
assert.True(messageID != "", "outboxStore.Delete: messageID is required")
if err := s.deleteMessage(ctx, messageID); err != nil {
pkg/storage/secret/metadata/outbox_store_test.go
+7 -3
View File
@@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/storage/secret/database"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
)
type outboxStoreModel struct {
@@ -110,10 +111,11 @@ func TestOutboxStoreSecureValueOperationInProgress(t *testing.T) {
t.Parallel()
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
tracer := noop.NewTracerProvider().Tracer("test")
ctx := context.Background()
outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB))
outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB, tracer), tracer)
_, err := outbox.Append(ctx, contracts.AppendOutboxMessage{
RequestID: "1",
@@ -142,10 +144,11 @@ func TestOutboxStoreSecureValueOperationInProgress(t *testing.T) {
func TestOutboxStore(t *testing.T) {
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
tracer := noop.NewTracerProvider().Tracer("test")
ctx := context.Background()
outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB))
outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB, tracer), tracer)
m1 := contracts.AppendOutboxMessage{
Type: contracts.CreateSecretOutboxMessage,
@@ -212,8 +215,9 @@ func TestOutboxStoreProperty(t *testing.T) {
// The number of iterations was decided arbitrarily based on the time the test takes to run
for range 10 {
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
tracer := noop.NewTracerProvider().Tracer("test")
outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB))
outbox := ProvideOutboxQueue(database.ProvideDatabase(testDB, tracer), tracer)
model := newOutboxStoreModel()
pkg/storage/secret/metadata/secure_value_store.go
+64 -2
View File
@@ -10,11 +10,13 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/storage/unified/sql"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
var _ contracts.SecureValueMetadataStorage = (*secureValueMetadataStorage)(nil)
func ProvideSecureValueMetadataStorage(db contracts.Database, features featuremgmt.FeatureToggles) (contracts.SecureValueMetadataStorage, error) {
func ProvideSecureValueMetadataStorage(db contracts.Database, tracer trace.Tracer, features featuremgmt.FeatureToggles) (contracts.SecureValueMetadataStorage, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) ||
!features.IsEnabledGlobally(featuremgmt.FlagSecretsManagementAppPlatform) {
return &secureValueMetadataStorage{}, nil
@@ -23,6 +25,7 @@ func ProvideSecureValueMetadataStorage(db contracts.Database, features featuremg
return &secureValueMetadataStorage{
db: db,
dialect: sqltemplate.DialectForDriver(db.DriverName()),
tracer: tracer,
}, nil
}
@@ -30,9 +33,17 @@ func ProvideSecureValueMetadataStorage(db contracts.Database, features featuremg
type secureValueMetadataStorage struct {
db contracts.Database
dialect sqltemplate.Dialect
tracer trace.Tracer
}
func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv0alpha1.SecureValue, actorUID string) (*secretv0alpha1.SecureValue, error) {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Create", trace.WithAttributes(
attribute.String("name", sv.GetName()),
attribute.String("namespace", sv.GetNamespace()),
attribute.String("actorUID", actorUID),
))
defer span.End()
sv.Status.Phase = secretv0alpha1.SecureValuePhasePending
sv.Status.Message = "Creating secure value"
@@ -110,6 +121,13 @@ func (s *secureValueMetadataStorage) Create(ctx context.Context, sv *secretv0alp
}
func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.Namespace, name string, opts contracts.ReadOpts) (*secretv0alpha1.SecureValue, error) {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Read", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
attribute.Bool("isForUpdate", opts.ForUpdate),
))
defer span.End()
secureValue, err := s.read(ctx, namespace, name, opts)
if err != nil {
return nil, err
@@ -124,6 +142,13 @@ func (s *secureValueMetadataStorage) Read(ctx context.Context, namespace xkube.N
}
func (s *secureValueMetadataStorage) Update(ctx context.Context, newSecureValue *secretv0alpha1.SecureValue, actorUID string) (*secretv0alpha1.SecureValue, error) {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Update", trace.WithAttributes(
attribute.String("name", newSecureValue.GetName()),
attribute.String("namespace", newSecureValue.GetNamespace()),
attribute.String("actorUID", actorUID),
))
defer span.End()
var newRow *secureValueDB
err := s.db.Transaction(ctx, func(ctx context.Context) error {
@@ -207,6 +232,12 @@ func (s *secureValueMetadataStorage) Update(ctx context.Context, newSecureValue
}
func (s *secureValueMetadataStorage) Delete(ctx context.Context, namespace xkube.Namespace, name string) error {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.Delete", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
))
defer span.End()
req := deleteSecureValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
@@ -230,7 +261,16 @@ func (s *secureValueMetadataStorage) Delete(ctx context.Context, namespace xkube
return nil
}
func (s *secureValueMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) ([]secretv0alpha1.SecureValue, error) {
func (s *secureValueMetadataStorage) List(ctx context.Context, namespace xkube.Namespace) (svList []secretv0alpha1.SecureValue, error error) {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.List", trace.WithAttributes(
attribute.String("namespace", namespace.String()),
))
defer span.End()
defer func() {
span.SetAttributes(attribute.Int("returnedList.count", len(svList)))
}()
req := listSecureValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
@@ -280,6 +320,13 @@ func (s *secureValueMetadataStorage) List(ctx context.Context, namespace xkube.N
}
func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespace xkube.Namespace, name string, externalID contracts.ExternalID) error {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.SetExternalID", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
attribute.String("externalID", externalID.String()),
))
defer span.End()
req := updateExternalIdSecureValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
@@ -309,6 +356,15 @@ func (s *secureValueMetadataStorage) SetExternalID(ctx context.Context, namespac
}
func (s *secureValueMetadataStorage) SetStatus(ctx context.Context, namespace xkube.Namespace, name string, status secretv0alpha1.SecureValueStatus) error {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.SetStatus", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
attribute.String("status.phase", string(status.Phase)),
attribute.String("status.message", status.Message),
attribute.String("status.externalID", status.ExternalID),
))
defer span.End()
req := updateStatusSecureValue{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
@@ -339,6 +395,12 @@ func (s *secureValueMetadataStorage) SetStatus(ctx context.Context, namespace xk
}
func (s *secureValueMetadataStorage) ReadForDecrypt(ctx context.Context, namespace xkube.Namespace, name string) (*contracts.DecryptSecureValue, error) {
ctx, span := s.tracer.Start(ctx, "SecureValueMetadataStorage.ReadForDecrypt", trace.WithAttributes(
attribute.String("name", name),
attribute.String("namespace", namespace.String()),
))
defer span.End()
req := readSecureValueForDecrypt{
SQLTemplate: sqltemplate.New(s.dialect),
Namespace: namespace.String(),
pkg/storage/secret/metadata/secure_value_store_test.go
+5 -3
View File
@@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/storage/secret/database"
"github.com/grafana/grafana/pkg/storage/secret/migrator"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
)
func createTestKeeper(t *testing.T, ctx context.Context, keeperStorage contracts.KeeperMetadataStorage, name, namespace string) string {
@@ -36,16 +37,17 @@ func createTestKeeper(t *testing.T, ctx context.Context, keeperStorage contracts
func Test_SecureValueMetadataStorage_CreateAndRead(t *testing.T) {
ctx := context.Background()
testDB := sqlstore.NewTestStore(t, sqlstore.WithMigrator(migrator.New()))
db := database.ProvideDatabase(testDB)
tracer := noop.NewTracerProvider().Tracer("test")
db := database.ProvideDatabase(testDB, tracer)
features := featuremgmt.WithFeatures(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, featuremgmt.FlagSecretsManagementAppPlatform)
// Initialize the secure value storage
secureValueStorage, err := ProvideSecureValueMetadataStorage(db, features)
secureValueStorage, err := ProvideSecureValueMetadataStorage(db, tracer, features)
require.NoError(t, err)
// Initialize the keeper storage
keeperStorage, err := ProvideKeeperMetadataStorage(db, features)
keeperStorage, err := ProvideKeeperMetadataStorage(db, tracer, features)
require.NoError(t, err)
t.Run("create and read a secure value", func(t *testing.T) {
pkg/storage/unified/resource/event.go
+21
View File
@@ -2,6 +2,7 @@ package resource
import (
"context"
"fmt"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
@@ -26,6 +27,26 @@ type WriteEvent struct {
ObjectOld utils.GrafanaMetaAccessor
}
func (e *WriteEvent) Validate() error {
if e.Object == nil {
return fmt.Errorf("object is nil")
}
if e.Key == nil {
return fmt.Errorf("key is nil")
}
if e.Value == nil {
return fmt.Errorf("value is nil")
}
if e.Type == resourcepb.WatchEvent_UNKNOWN {
return fmt.Errorf("watch event type is unknown")
}
return nil
}
// WrittenEvent is a WriteEvent reported with a resource version.
type WrittenEvent struct {
Type resourcepb.WatchEvent_Type
pkg/storage/unified/resource/kv_test.go
+169 -206
View File
@@ -1,15 +1,13 @@
package resource
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"testing"
badger "github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -27,139 +25,9 @@ func setupTestBadgerDB(t *testing.T) *badger.DB {
func setupTestKV(t *testing.T) KV {
db := setupTestBadgerDB(t)
t.Cleanup(func() {
err := db.Close()
require.NoError(t, err)
})
return NewBadgerKV(db)
}
func TestBadgerKV_Get(t *testing.T) {
db := setupTestBadgerDB(t)
kv := NewBadgerKV(db)
ctx := context.Background()
// Setup test data
err := db.Update(func(txn *badger.Txn) error {
return txn.Set([]byte("section/key1"), []byte("value1"))
})
require.NoError(t, err)
t.Run("Get existing key", func(t *testing.T) {
obj, err := kv.Get(ctx, "section", "key1")
require.NoError(t, err)
assert.Equal(t, "key1", obj.Key)
// Read the value from the Reader
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, []byte("value1"), value)
})
t.Run("Get non-existent key", func(t *testing.T) {
_, err := kv.Get(ctx, "section", "nonexistent")
assert.Error(t, err)
assert.Equal(t, ErrNotFound, err)
})
}
func TestBadgerKV_Save(t *testing.T) {
db := setupTestBadgerDB(t)
kv := NewBadgerKV(db)
ctx := context.Background()
t.Run("Save new key", func(t *testing.T) {
err := kv.Save(ctx, "section", "key1", bytes.NewReader([]byte("value1")))
require.NoError(t, err)
// Verify the value was saved
obj, err := kv.Get(ctx, "section", "key1")
require.NoError(t, err)
assert.Equal(t, "key1", obj.Key)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, []byte("value1"), value)
})
t.Run("Save overwrite existing key", func(t *testing.T) {
// First save
err := kv.Save(ctx, "section", "key1", bytes.NewReader([]byte("oldvalue")))
require.NoError(t, err)
// Overwrite
err = kv.Save(ctx, "section", "key1", bytes.NewReader([]byte("newvalue")))
require.NoError(t, err)
// Verify the value was updated
obj, err := kv.Get(ctx, "section", "key1")
require.NoError(t, err)
assert.Equal(t, "key1", obj.Key)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, []byte("newvalue"), value)
})
}
func TestBadgerKV_Delete(t *testing.T) {
db := setupTestBadgerDB(t)
kv := NewBadgerKV(db)
ctx := context.Background()
t.Run("Delete existing key", func(t *testing.T) {
// First create a key
err := kv.Save(ctx, "section", "key1", bytes.NewReader([]byte("value1")))
require.NoError(t, err)
// Delete it
err = kv.Delete(ctx, "section", "key1")
require.NoError(t, err)
// Verify it's gone
_, err = kv.Get(ctx, "section", "key1")
assert.Error(t, err)
assert.Equal(t, ErrNotFound, err)
})
t.Run("Delete non-existent key", func(t *testing.T) {
err := kv.Delete(ctx, "section", "nonexistent")
assert.Error(t, err)
assert.Equal(t, ErrNotFound, err)
})
}
// setupIteratorTestData creates a test environment with common test data
func setupIteratorTestData(t *testing.T) (*badgerKV, context.Context) {
db := setupTestBadgerDB(t)
t.Cleanup(func() {
err := db.Close()
require.NoError(t, err)
})
kv := NewBadgerKV(db)
ctx := context.Background()
// Setup test data
keys := []string{"a1", "a2", "b1", "b2", "c1"}
for _, k := range keys {
err := kv.Save(ctx, "section", k, bytes.NewReader([]byte("value"+k)))
require.NoError(t, err)
}
return kv, ctx
}
// iteratorTestCase represents a test case for iteration methods
type iteratorTestCase struct {
name string
options ListOptions
expectedKeys []string
}
func TestPrefixRangeEnd(t *testing.T) {
require.Equal(t, "b", PrefixRangeEnd("a"))
require.Equal(t, "a/c", PrefixRangeEnd("a/b"))
@@ -167,95 +35,190 @@ func TestPrefixRangeEnd(t *testing.T) {
require.Equal(t, "", PrefixRangeEnd(""))
}
func TestBadgerKV_Keys(t *testing.T) {
for _, tc := range []iteratorTestCase{
{
name: "all items",
options: ListOptions{},
expectedKeys: []string{"a1", "a2", "b1", "b2", "c1"},
},
{
name: "with limit",
options: ListOptions{Limit: 2},
expectedKeys: []string{"a1", "a2"},
},
{
name: "with range",
options: ListOptions{StartKey: "a", EndKey: "b"},
expectedKeys: []string{"a1", "a2"},
},
{
name: "with prefix",
options: ListOptions{StartKey: "a", EndKey: PrefixRangeEnd("a")},
expectedKeys: []string{"a1", "a2"},
},
{
name: "in descending order",
options: ListOptions{Sort: SortOrderDesc},
expectedKeys: []string{"c1", "b2", "b1", "a2", "a1"},
},
{
name: "in descending order with prefix",
options: ListOptions{StartKey: "a", EndKey: PrefixRangeEnd("a"), Sort: SortOrderDesc},
expectedKeys: []string{"a2", "a1"},
},
} {
t.Run("Keys "+tc.name, func(t *testing.T) {
kv, ctx := setupIteratorTestData(t)
func TestBadgerKVSmoke(t *testing.T) {
// Simple smoke test to ensure the basic badger KV implementation works
kv := setupTestKV(t)
ctx := context.Background()
var keys []string
for k, err := range kv.Keys(ctx, "section", tc.options) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, tc.expectedKeys, keys)
})
}
// Test unix timestamp works
timestamp, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
require.Greater(t, timestamp, int64(0))
// Test get non-existent key returns proper error
_, err = kv.Get(ctx, "test-section", "non-existent")
require.Error(t, err)
require.Equal(t, ErrNotFound, err)
}
func TestBadgerKV_Concurrent(t *testing.T) {
func TestBadgerKV_UnderlyingStorage(t *testing.T) {
// Test internal key storage format and structure
db := setupTestBadgerDB(t)
kv := NewBadgerKV(db)
ctx := context.Background()
t.Run("Concurrent operations", func(t *testing.T) {
const numGoroutines = 10
done := make(chan struct{})
t.Run("keys are stored with section prefix", func(t *testing.T) {
section := "test-section"
key := "test-key"
value := "test-value"
expectedInternalKey := section + "/" + key
for i := 0; i < numGoroutines; i++ {
go func(i int) {
defer func() { done <- struct{}{} }()
// Save through KV interface
err := kv.Save(ctx, section, key, strings.NewReader(value))
require.NoError(t, err)
key := fmt.Sprintf("key%d", i)
value := []byte(fmt.Sprintf("value%d", i))
// Verify the raw key exists in badger with correct format
err = db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(expectedInternalKey))
require.NoError(t, err)
// Save
err := kv.Save(ctx, "section", key, bytes.NewReader(value))
require.NoError(t, err)
// Verify the value is correct
valueBytes, err := item.ValueCopy(nil)
require.NoError(t, err)
require.Equal(t, value, string(valueBytes))
// Get
obj, err := kv.Get(ctx, "section", key)
require.NoError(t, err)
assert.Equal(t, key, obj.Key)
return nil
})
require.NoError(t, err)
})
readValue, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, value, readValue)
t.Run("sections are properly isolated", func(t *testing.T) {
section1 := "section1"
section2 := "section2"
key := "same-key"
value1 := "value-from-section1"
value2 := "value-from-section2"
// Delete
err = kv.Delete(ctx, "section", key)
require.NoError(t, err)
// Save same key in different sections
err := kv.Save(ctx, section1, key, strings.NewReader(value1))
require.NoError(t, err)
err = kv.Save(ctx, section2, key, strings.NewReader(value2))
require.NoError(t, err)
// Verify deleted
_, err = kv.Get(ctx, "section", key)
assert.Error(t, err)
}(i)
// Verify both keys exist in badger with different internal keys
err = db.View(func(txn *badger.Txn) error {
// Check section1 key
item1, err := txn.Get([]byte(section1 + "/" + key))
require.NoError(t, err)
value1Bytes, err := item1.ValueCopy(nil)
require.NoError(t, err)
require.Equal(t, value1, string(value1Bytes))
// Check section2 key
item2, err := txn.Get([]byte(section2 + "/" + key))
require.NoError(t, err)
value2Bytes, err := item2.ValueCopy(nil)
require.NoError(t, err)
require.Equal(t, value2, string(value2Bytes))
return nil
})
require.NoError(t, err)
// Verify KV interface returns correct values for each section
obj1, err := kv.Get(ctx, section1, key)
require.NoError(t, err)
val1, err := io.ReadAll(obj1.Value)
require.NoError(t, err)
require.Equal(t, value1, string(val1))
err = obj1.Value.Close()
require.NoError(t, err)
obj2, err := kv.Get(ctx, section2, key)
require.NoError(t, err)
val2, err := io.ReadAll(obj2.Value)
require.NoError(t, err)
require.Equal(t, value2, string(val2))
err = obj2.Value.Close()
require.NoError(t, err)
})
t.Run("delete removes correct internal key", func(t *testing.T) {
section := "delete-section"
key := "delete-key"
value := "delete-value"
internalKey := section + "/" + key
// Save and verify it exists
err := kv.Save(ctx, section, key, strings.NewReader(value))
require.NoError(t, err)
// Verify it exists in badger
err = db.View(func(txn *badger.Txn) error {
_, err := txn.Get([]byte(internalKey))
return err
})
require.NoError(t, err)
// Delete through KV interface
err = kv.Delete(ctx, section, key)
require.NoError(t, err)
// Verify it's gone from badger
err = db.View(func(txn *badger.Txn) error {
_, err := txn.Get([]byte(internalKey))
return err
})
require.Error(t, err)
require.Equal(t, badger.ErrKeyNotFound, err)
})
t.Run("keys iteration respects section boundaries", func(t *testing.T) {
section1 := "alpha"
section2 := "beta"
// Add keys to both sections
keys1 := []string{"a1", "a2", "a3"}
keys2 := []string{"b1", "b2", "b3"}
for _, k := range keys1 {
err := kv.Save(ctx, section1, k, strings.NewReader("value"+k))
require.NoError(t, err)
}
for _, k := range keys2 {
err := kv.Save(ctx, section2, k, strings.NewReader("value"+k))
require.NoError(t, err)
}
// Wait for all goroutines to complete
for i := 0; i < numGoroutines; i++ {
<-done
// List keys from section1 only
var foundKeys1 []string
for k, err := range kv.Keys(ctx, section1, ListOptions{}) {
require.NoError(t, err)
foundKeys1 = append(foundKeys1, k)
}
require.Equal(t, keys1, foundKeys1)
// List keys from section2 only
var foundKeys2 []string
for k, err := range kv.Keys(ctx, section2, ListOptions{}) {
require.NoError(t, err)
foundKeys2 = append(foundKeys2, k)
}
require.Equal(t, keys2, foundKeys2)
// Verify raw badger contains all keys with proper prefixes
var allRawKeys []string
err := db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false
iter := txn.NewIterator(opts)
defer iter.Close()
for iter.Rewind(); iter.Valid(); iter.Next() {
item := iter.Item()
allRawKeys = append(allRawKeys, string(item.Key()))
}
return nil
})
require.NoError(t, err)
// Check that all expected internal keys exist
expectedInternalKeys := []string{
"alpha/a1", "alpha/a2", "alpha/a3",
"beta/b1", "beta/b2", "beta/b3",
}
for _, expectedKey := range expectedInternalKeys {
require.Contains(t, allRawKeys, expectedKey, "Expected internal key %s should exist", expectedKey)
}
})
}
pkg/storage/unified/resource/storage_backend.go
+798
View File
@@ -0,0 +1,798 @@
package resource
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math/rand/v2"
"net/http"
"sort"
"strings"
"time"
"github.com/bwmarrin/snowflake"
"github.com/grafana/grafana-app-sdk/logging"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
defaultListBufferSize = 100
)
// Unified storage backend based on KV storage.
type kvStorageBackend struct {
snowflake *snowflake.Node
kv KV
dataStore *dataStore
metaStore *metadataStore
eventStore *eventStore
notifier *notifier
builder DocumentBuilder
log logging.Logger
}
var _ StorageBackend = &kvStorageBackend{}
func NewKvStorageBackend(kv KV) *kvStorageBackend {
s, err := snowflake.NewNode(rand.Int64N(1024))
if err != nil {
panic(err)
}
eventStore := newEventStore(kv)
return &kvStorageBackend{
kv: kv,
dataStore: newDataStore(kv),
metaStore: newMetadataStore(kv),
eventStore: eventStore,
notifier: newNotifier(eventStore, notifierOptions{}),
snowflake: s,
builder: StandardDocumentBuilder(), // For now we use the standard document builder.
log: &logging.NoOpLogger{}, // Make this configurable
}
}
// WriteEvent writes a resource event (create/update/delete) to the storage backend.
func (k *kvStorageBackend) WriteEvent(ctx context.Context, event WriteEvent) (int64, error) {
if err := event.Validate(); err != nil {
return 0, fmt.Errorf("invalid event: %w", err)
}
rv := k.snowflake.Generate().Int64()
// Write data.
var action DataAction
switch event.Type {
case resourcepb.WatchEvent_ADDED:
action = DataActionCreated
// Check if resource already exists for create operations
_, err := k.metaStore.GetLatestResourceKey(ctx, MetaGetRequestKey{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
})
if err == nil {
// Resource exists, return already exists error
return 0, ErrResourceAlreadyExists
}
if !errors.Is(err, ErrNotFound) {
// Some other error occurred
return 0, fmt.Errorf("failed to check if resource exists: %w", err)
}
case resourcepb.WatchEvent_MODIFIED:
action = DataActionUpdated
case resourcepb.WatchEvent_DELETED:
action = DataActionDeleted
default:
return 0, fmt.Errorf("invalid event type: %d", event.Type)
}
// Build the search document
doc, err := k.builder.BuildDocument(ctx, event.Key, rv, event.Value)
if err != nil {
return 0, fmt.Errorf("failed to build document: %w", err)
}
// Write the data
err = k.dataStore.Save(ctx, DataKey{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
ResourceVersion: rv,
Action: action,
}, bytes.NewReader(event.Value))
if err != nil {
return 0, fmt.Errorf("failed to write data: %w", err)
}
// Write metadata
err = k.metaStore.Save(ctx, MetaDataObj{
Key: MetaDataKey{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
ResourceVersion: rv,
Action: action,
Folder: event.Object.GetFolder(),
},
Value: MetaData{
IndexableDocument: *doc,
},
})
if err != nil {
return 0, fmt.Errorf("failed to write metadata: %w", err)
}
// Write event
err = k.eventStore.Save(ctx, Event{
Namespace: event.Key.Namespace,
Group: event.Key.Group,
Resource: event.Key.Resource,
Name: event.Key.Name,
ResourceVersion: rv,
Action: action,
Folder: event.Object.GetFolder(),
PreviousRV: event.PreviousRV,
})
if err != nil {
return 0, fmt.Errorf("failed to save event: %w", err)
}
return rv, nil
}
func (k *kvStorageBackend) ReadResource(ctx context.Context, req *resourcepb.ReadRequest) *BackendReadResponse {
if req.Key == nil {
return &BackendReadResponse{Error: &resourcepb.ErrorResult{Code: http.StatusBadRequest, Message: "missing key"}}
}
meta, err := k.metaStore.GetResourceKeyAtRevision(ctx, MetaGetRequestKey{
Namespace: req.Key.Namespace,
Group: req.Key.Group,
Resource: req.Key.Resource,
Name: req.Key.Name,
}, req.ResourceVersion)
if errors.Is(err, ErrNotFound) {
return &BackendReadResponse{Error: &resourcepb.ErrorResult{Code: http.StatusNotFound, Message: "not found"}}
} else if err != nil {
return &BackendReadResponse{Error: &resourcepb.ErrorResult{Code: http.StatusInternalServerError, Message: err.Error()}}
}
data, err := k.dataStore.Get(ctx, DataKey{
Namespace: req.Key.Namespace,
Group: req.Key.Group,
Resource: req.Key.Resource,
Name: req.Key.Name,
ResourceVersion: meta.ResourceVersion,
Action: meta.Action,
})
if err != nil || data == nil {
return &BackendReadResponse{Error: &resourcepb.ErrorResult{Code: http.StatusInternalServerError, Message: err.Error()}}
}
value, err := readAndClose(data)
if err != nil {
return &BackendReadResponse{Error: &resourcepb.ErrorResult{Code: http.StatusInternalServerError, Message: err.Error()}}
}
return &BackendReadResponse{
Key: req.Key,
ResourceVersion: meta.ResourceVersion,
Value: value,
Folder: meta.Folder,
}
}
// ListIterator returns an iterator for listing resources.
func (k *kvStorageBackend) ListIterator(ctx context.Context, req *resourcepb.ListRequest, cb func(ListIterator) error) (int64, error) {
if req.Options == nil || req.Options.Key == nil {
return 0, fmt.Errorf("missing options or key in ListRequest")
}
// Parse continue token if provided
offset := int64(0)
resourceVersion := req.ResourceVersion
if req.NextPageToken != "" {
token, err := GetContinueToken(req.NextPageToken)
if err != nil {
return 0, fmt.Errorf("invalid continue token: %w", err)
}
offset = token.StartOffset
resourceVersion = token.ResourceVersion
}
// We set the listRV to the current time.
listRV := k.snowflake.Generate().Int64()
if resourceVersion > 0 {
listRV = resourceVersion
}
// Fetch the latest objects
keys := make([]MetaDataKey, 0, min(defaultListBufferSize, req.Limit+1))
idx := 0
for metaKey, err := range k.metaStore.ListResourceKeysAtRevision(ctx, MetaListRequestKey{
Namespace: req.Options.Key.Namespace,
Group: req.Options.Key.Group,
Resource: req.Options.Key.Resource,
Name: req.Options.Key.Name,
}, resourceVersion) {
if err != nil {
return 0, err
}
// Skip the first offset items. This is not efficient, but it's a simple way to implement it for now.
if idx < int(offset) {
idx++
continue
}
keys = append(keys, metaKey)
// Only fetch the first limit items + 1 to get the next token.
if len(keys) >= int(req.Limit+1) {
break
}
}
iter := kvListIterator{
keys: keys,
currentIndex: -1,
ctx: ctx,
listRV: listRV,
offset: offset,
limit: req.Limit + 1, // TODO: for now we need at least one more item. Fix the caller
dataStore: k.dataStore,
}
err := cb(&iter)
if err != nil {
return 0, err
}
return listRV, nil
}
// kvListIterator implements ListIterator for KV storage
type kvListIterator struct {
ctx context.Context
keys []MetaDataKey
currentIndex int
dataStore *dataStore
listRV int64
offset int64
limit int64
// current
rv int64
err error
value []byte
}
func (i *kvListIterator) Next() bool {
i.currentIndex++
if i.currentIndex >= len(i.keys) {
return false
}
if int64(i.currentIndex) >= i.limit {
return false
}
i.rv, i.err = i.keys[i.currentIndex].ResourceVersion, nil
data, err := i.dataStore.Get(i.ctx, DataKey{
Namespace: i.keys[i.currentIndex].Namespace,
Group: i.keys[i.currentIndex].Group,
Resource: i.keys[i.currentIndex].Resource,
Name: i.keys[i.currentIndex].Name,
ResourceVersion: i.keys[i.currentIndex].ResourceVersion,
Action: i.keys[i.currentIndex].Action,
})
if err != nil {
i.err = err
return false
}
i.value, i.err = readAndClose(data)
if i.err != nil {
return false
}
// increment the offset
i.offset++
return true
}
func (i *kvListIterator) Error() error {
return nil
}
func (i *kvListIterator) ContinueToken() string {
return ContinueToken{
StartOffset: i.offset,
ResourceVersion: i.listRV,
}.String()
}
func (i *kvListIterator) ResourceVersion() int64 {
return i.rv
}
func (i *kvListIterator) Namespace() string {
return i.keys[i.currentIndex].Namespace
}
func (i *kvListIterator) Name() string {
return i.keys[i.currentIndex].Name
}
func (i *kvListIterator) Folder() string {
return i.keys[i.currentIndex].Folder
}
func (i *kvListIterator) Value() []byte {
return i.value
}
func validateListHistoryRequest(req *resourcepb.ListRequest) error {
if req.Options == nil || req.Options.Key == nil {
return fmt.Errorf("missing options or key in ListRequest")
}
key := req.Options.Key
if key.Group == "" {
return fmt.Errorf("group is required")
}
if key.Resource == "" {
return fmt.Errorf("resource is required")
}
if key.Namespace == "" {
return fmt.Errorf("namespace is required")
}
if key.Name == "" {
return fmt.Errorf("name is required")
}
return nil
}
// filterHistoryKeysByVersion filters history keys based on version match criteria
func filterHistoryKeysByVersion(historyKeys []DataKey, req *resourcepb.ListRequest) ([]DataKey, error) {
switch req.GetVersionMatchV2() {
case resourcepb.ResourceVersionMatchV2_Exact:
if req.ResourceVersion <= 0 {
return nil, fmt.Errorf("expecting an explicit resource version query when using Exact matching")
}
var exactKeys []DataKey
for _, key := range historyKeys {
if key.ResourceVersion == req.ResourceVersion {
exactKeys = append(exactKeys, key)
}
}
return exactKeys, nil
case resourcepb.ResourceVersionMatchV2_NotOlderThan:
if req.ResourceVersion > 0 {
var filteredKeys []DataKey
for _, key := range historyKeys {
if key.ResourceVersion >= req.ResourceVersion {
filteredKeys = append(filteredKeys, key)
}
}
return filteredKeys, nil
}
default:
if req.ResourceVersion > 0 {
var filteredKeys []DataKey
for _, key := range historyKeys {
if key.ResourceVersion <= req.ResourceVersion {
filteredKeys = append(filteredKeys, key)
}
}
return filteredKeys, nil
}
}
return historyKeys, nil
}
// applyLiveHistoryFilter applies "live" history logic by ignoring events before the last delete
func applyLiveHistoryFilter(filteredKeys []DataKey, req *resourcepb.ListRequest) []DataKey {
useLatestDeletionAsMinRV := req.ResourceVersion == 0 && req.Source != resourcepb.ListRequest_TRASH && req.GetVersionMatchV2() != resourcepb.ResourceVersionMatchV2_Exact
if !useLatestDeletionAsMinRV {
return filteredKeys
}
latestDeleteRV := int64(0)
for _, key := range filteredKeys {
if key.Action == DataActionDeleted && key.ResourceVersion > latestDeleteRV {
latestDeleteRV = key.ResourceVersion
}
}
if latestDeleteRV > 0 {
var liveKeys []DataKey
for _, key := range filteredKeys {
if key.ResourceVersion > latestDeleteRV {
liveKeys = append(liveKeys, key)
}
}
return liveKeys
}
return filteredKeys
}
// sortByResourceVersion sorts the history keys based on the sortAscending flag
func sortByResourceVersion(filteredKeys []DataKey, sortAscending bool) {
if sortAscending {
sort.Slice(filteredKeys, func(i, j int) bool {
return filteredKeys[i].ResourceVersion < filteredKeys[j].ResourceVersion
})
} else {
sort.Slice(filteredKeys, func(i, j int) bool {
return filteredKeys[i].ResourceVersion > filteredKeys[j].ResourceVersion
})
}
}
// applyPagination filters keys based on pagination parameters
func applyPagination(keys []DataKey, lastSeenRV int64, sortAscending bool) []DataKey {
if lastSeenRV == 0 {
return keys
}
var pagedKeys []DataKey
for _, key := range keys {
if sortAscending && key.ResourceVersion > lastSeenRV {
pagedKeys = append(pagedKeys, key)
} else if !sortAscending && key.ResourceVersion < lastSeenRV {
pagedKeys = append(pagedKeys, key)
}
}
return pagedKeys
}
// ListHistory is like ListIterator, but it returns the history of a resource.
func (k *kvStorageBackend) ListHistory(ctx context.Context, req *resourcepb.ListRequest, fn func(ListIterator) error) (int64, error) {
if err := validateListHistoryRequest(req); err != nil {
return 0, err
}
key := req.Options.Key
// Parse continue token if provided
lastSeenRV := int64(0)
sortAscending := req.GetVersionMatchV2() == resourcepb.ResourceVersionMatchV2_NotOlderThan
if req.NextPageToken != "" {
token, err := GetContinueToken(req.NextPageToken)
if err != nil {
return 0, fmt.Errorf("invalid continue token: %w", err)
}
lastSeenRV = token.ResourceVersion
sortAscending = token.SortAscending
}
// Generate a new resource version for the list
listRV := k.snowflake.Generate().Int64()
// Get all history entries by iterating through datastore keys
historyKeys := make([]DataKey, 0, min(defaultListBufferSize, req.Limit+1))
// Use datastore.Keys to get all data keys for this specific resource
for dataKey, err := range k.dataStore.Keys(ctx, ListRequestKey{
Namespace: key.Namespace,
Group: key.Group,
Resource: key.Resource,
Name: key.Name,
}) {
if err != nil {
return 0, err
}
historyKeys = append(historyKeys, dataKey)
}
// Check if context has been cancelled
if ctx.Err() != nil {
return 0, ctx.Err()
}
// Handle trash differently from regular history
if req.Source == resourcepb.ListRequest_TRASH {
return k.processTrashEntries(ctx, req, fn, historyKeys, lastSeenRV, sortAscending, listRV)
}
// Apply filtering based on version match
filteredKeys, filterErr := filterHistoryKeysByVersion(historyKeys, req)
if filterErr != nil {
return 0, filterErr
}
// Apply "live" history logic: ignore events before the last delete
filteredKeys = applyLiveHistoryFilter(filteredKeys, req)
// Sort the entries if not already sorted correctly
sortByResourceVersion(filteredKeys, sortAscending)
// Pagination: filter out items up to and including lastSeenRV
pagedKeys := applyPagination(filteredKeys, lastSeenRV, sortAscending)
iter := kvHistoryIterator{
keys: pagedKeys,
currentIndex: -1,
ctx: ctx,
listRV: listRV,
sortAscending: sortAscending,
dataStore: k.dataStore,
}
err := fn(&iter)
if err != nil {
return 0, err
}
return listRV, nil
}
// processTrashEntries handles the special case of listing deleted items (trash)
func (k *kvStorageBackend) processTrashEntries(ctx context.Context, req *resourcepb.ListRequest, fn func(ListIterator) error, historyKeys []DataKey, lastSeenRV int64, sortAscending bool, listRV int64) (int64, error) {
// Filter to only deleted entries
var deletedKeys []DataKey
for _, key := range historyKeys {
if key.Action == DataActionDeleted {
deletedKeys = append(deletedKeys, key)
}
}
// Check if the resource currently exists (is live)
// If it exists, don't return any trash entries
_, err := k.metaStore.GetLatestResourceKey(ctx, MetaGetRequestKey{
Namespace: req.Options.Key.Namespace,
Group: req.Options.Key.Group,
Resource: req.Options.Key.Resource,
Name: req.Options.Key.Name,
})
var trashKeys []DataKey
if errors.Is(err, ErrNotFound) {
// Resource doesn't exist currently, so we can return the latest delete
// Find the latest delete event
var latestDelete *DataKey
for _, key := range deletedKeys {
if latestDelete == nil || key.ResourceVersion > latestDelete.ResourceVersion {
latestDelete = &key
}
}
if latestDelete != nil {
trashKeys = append(trashKeys, *latestDelete)
}
}
// If err != ErrNotFound, the resource exists, so no trash entries should be returned
// Apply version filtering
filteredKeys, err := filterHistoryKeysByVersion(trashKeys, req)
if err != nil {
return 0, err
}
// Sort the entries
sortByResourceVersion(filteredKeys, sortAscending)
// Pagination: filter out items up to and including lastSeenRV
pagedKeys := applyPagination(filteredKeys, lastSeenRV, sortAscending)
iter := kvHistoryIterator{
keys: pagedKeys,
currentIndex: -1,
ctx: ctx,
listRV: listRV,
sortAscending: sortAscending,
dataStore: k.dataStore,
}
err = fn(&iter)
if err != nil {
return 0, err
}
return listRV, nil
}
// kvHistoryIterator implements ListIterator for KV storage history
type kvHistoryIterator struct {
ctx context.Context
keys []DataKey
currentIndex int
listRV int64
sortAscending bool
dataStore *dataStore
// current
rv int64
err error
value []byte
folder string
}
func (i *kvHistoryIterator) Next() bool {
i.currentIndex++
if i.currentIndex >= len(i.keys) {
return false
}
key := i.keys[i.currentIndex]
i.rv = key.ResourceVersion
// Read the value from the ReadCloser
data, err := i.dataStore.Get(i.ctx, key)
if err != nil {
i.err = err
return false
}
if data == nil {
i.err = fmt.Errorf("data is nil")
return false
}
i.value, i.err = readAndClose(data)
if i.err != nil {
return false
}
// Extract the folder from the meta data
partial := &metav1.PartialObjectMetadata{}
err = json.Unmarshal(i.value, partial)
if err != nil {
i.err = err
return false
}
meta, err := utils.MetaAccessor(partial)
if err != nil {
i.err = err
return false
}
i.folder = meta.GetFolder()
i.err = nil
return true
}
func (i *kvHistoryIterator) Error() error {
return i.err
}
func (i *kvHistoryIterator) ContinueToken() string {
if i.currentIndex < 0 || i.currentIndex >= len(i.keys) {
return ""
}
token := ContinueToken{
StartOffset: i.rv,
ResourceVersion: i.keys[i.currentIndex].ResourceVersion,
SortAscending: i.sortAscending,
}
return token.String()
}
func (i *kvHistoryIterator) ResourceVersion() int64 {
return i.rv
}
func (i *kvHistoryIterator) Namespace() string {
if i.currentIndex >= 0 && i.currentIndex < len(i.keys) {
return i.keys[i.currentIndex].Namespace
}
return ""
}
func (i *kvHistoryIterator) Name() string {
if i.currentIndex >= 0 && i.currentIndex < len(i.keys) {
return i.keys[i.currentIndex].Name
}
return ""
}
func (i *kvHistoryIterator) Folder() string {
return i.folder
}
func (i *kvHistoryIterator) Value() []byte {
return i.value
}
// WatchWriteEvents returns a channel that receives write events.
func (k *kvStorageBackend) WatchWriteEvents(ctx context.Context) (<-chan *WrittenEvent, error) {
// Create a channel to receive events
events := make(chan *WrittenEvent, 10000) // TODO: make this configurable
notifierEvents := k.notifier.Watch(ctx, defaultWatchOptions())
go func() {
for event := range notifierEvents {
// fetch the data
dataReader, err := k.dataStore.Get(ctx, DataKey{
Namespace: event.Namespace,
Group: event.Group,
Resource: event.Resource,
Name: event.Name,
ResourceVersion: event.ResourceVersion,
Action: event.Action,
})
if err != nil || dataReader == nil {
k.log.Error("failed to get data for event", "error", err)
continue
}
data, err := readAndClose(dataReader)
if err != nil {
k.log.Error("failed to read and close data for event", "error", err)
continue
}
var t resourcepb.WatchEvent_Type
switch event.Action {
case DataActionCreated:
t = resourcepb.WatchEvent_ADDED
case DataActionUpdated:
t = resourcepb.WatchEvent_MODIFIED
case DataActionDeleted:
t = resourcepb.WatchEvent_DELETED
}
events <- &WrittenEvent{
Key: &resourcepb.ResourceKey{
Namespace: event.Namespace,
Group: event.Group,
Resource: event.Resource,
Name: event.Name,
},
Type: t,
Folder: event.Folder,
Value: data,
ResourceVersion: event.ResourceVersion,
PreviousRV: event.PreviousRV,
Timestamp: event.ResourceVersion / time.Second.Nanoseconds(), // convert to seconds
}
}
close(events)
}()
return events, nil
}
// GetResourceStats returns resource stats within the storage backend.
// TODO: this isn't very efficient, we should use a more efficient algorithm.
func (k *kvStorageBackend) GetResourceStats(ctx context.Context, namespace string, minCount int) ([]ResourceStats, error) {
stats := make([]ResourceStats, 0)
res := make(map[string]map[string]bool)
rvs := make(map[string]int64)
// Use datastore.Keys to get all data keys for the namespace
for dataKey, err := range k.dataStore.Keys(ctx, ListRequestKey{Namespace: namespace}) {
if err != nil {
return nil, err
}
key := fmt.Sprintf("%s/%s/%s", dataKey.Namespace, dataKey.Group, dataKey.Resource)
if _, ok := res[key]; !ok {
res[key] = make(map[string]bool)
rvs[key] = 1
}
res[key][dataKey.Name] = dataKey.Action != DataActionDeleted
rvs[key] = dataKey.ResourceVersion
}
for key, names := range res {
parts := strings.Split(key, "/")
count := int64(0)
for _, exists := range names {
if exists {
count++
}
}
if count <= int64(minCount) {
continue
}
stats = append(stats, ResourceStats{
NamespacedResource: NamespacedResource{
Namespace: parts[0],
Group: parts[1],
Resource: parts[2],
},
Count: count,
ResourceVersion: rvs[key],
})
}
return stats, nil
}
// readAndClose reads all data from a ReadCloser and ensures it's closed,
// combining any errors from both operations.
func readAndClose(r io.ReadCloser) ([]byte, error) {
data, err := io.ReadAll(r)
return data, errors.Join(err, r.Close())
}
pkg/storage/unified/resource/storage_backend_test.go
+1007
View File
File diff suppressed because it is too large Load Diff
pkg/storage/unified/sql/db/dbimpl/db_engine.go
+6
View File
@@ -38,6 +38,12 @@ func getEngine(cfg *setting.Cfg) (*xorm.Engine, error) {
if err != nil {
return nil, fmt.Errorf("open database: %w", err)
}
engine.SetMaxOpenConns(dbSection.Key("max_open_conn").MustInt(0))
engine.SetMaxIdleConns(dbSection.Key("max_idle_conn").MustInt(4))
maxLifetime := time.Duration(dbSection.Key("conn_max_lifetime").MustInt(14400)) * time.Second
engine.SetConnMaxLifetime(maxLifetime)
return engine, nil
default:
return nil, fmt.Errorf("unsupported database type: %s", dbType)
pkg/storage/unified/sql/db/dbimpl/dbimpl.go
+18 -1
View File
@@ -78,9 +78,10 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.
getter := newConfGetter(cfg.SectionWithEnvOverrides("resource_api"), "db_")
fallbackGetter := newConfGetter(cfg.SectionWithEnvOverrides("database"), "")
logger := log.New("entity-db")
p = &resourceDBProvider{
cfg: cfg,
log: log.New("entity-db"),
log: logger,
logQueries: getter.Bool("log_queries"),
migrateFunc: migrations.MigrateResourceStore,
tracer: tracer,
@@ -91,11 +92,13 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.
switch {
// Deprecated: First try with the config in the "resource_api" section, which is specific to Unified Storage
case dbType == dbTypePostgres:
logger.Info("Using resource_api section", "db_type", dbType)
p.registerMetrics = true
p.engine, err = getEnginePostgres(getter)
return p, err
case dbType == dbTypeMySQL:
logger.Info("Using resource_api section", "db_type", dbType)
p.registerMetrics = true
p.engine, err = getEngineMySQL(getter)
return p, err
@@ -105,6 +108,7 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.
// If we have an empty Resource API db config, try with the core Grafana database config
case grafanaDBType != "":
logger.Info("Using database section", "db_type", grafanaDBType)
p.registerMetrics = true
p.engine, err = getEngine(cfg)
return p, err
@@ -121,6 +125,19 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.
}
func (p *resourceDBProvider) init(ctx context.Context) (db.DB, error) {
p.log.Info("Initializing Resource DB",
"db_type",
p.engine.Dialect().DriverName(),
"open_conn",
p.engine.DB().DB.Stats().OpenConnections,
"in_use_conn",
p.engine.DB().DB.Stats().InUse,
"idle_conn",
p.engine.DB().DB.Stats().Idle,
"max_open_conn",
p.engine.DB().DB.Stats().MaxOpenConnections,
)
if p.registerMetrics {
err := prometheus.Register(sqlstats.NewStatsCollector("unified_storage", p.engine.DB().DB))
if err != nil {
pkg/storage/unified/testing/kv.go
+490
View File
@@ -0,0 +1,490 @@
package test
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/util/testutil"
)
// Test names for the KV test suite
const (
TestKVGet = "get operations"
TestKVSave = "save operations"
TestKVDelete = "delete operations"
TestKVKeys = "keys listing"
TestKVKeysWithLimits = "keys with limits and ranges"
TestKVKeysWithSort = "keys with sorting"
TestKVConcurrent = "concurrent operations"
TestKVUnixTimestamp = "unix timestamp"
)
// NewKVFunc is a function that creates a new KV instance for testing
type NewKVFunc func(ctx context.Context) resource.KV
// KVTestOptions configures which tests to run
type KVTestOptions struct {
NSPrefix string // namespace prefix for isolation
}
// GenerateRandomKVPrefix creates a random namespace prefix for test isolation
func GenerateRandomKVPrefix() string {
return fmt.Sprintf("kvtest-%d", time.Now().UnixNano())
}
// RunKVTest runs the KV test suite
func RunKVTest(t *testing.T, newKV NewKVFunc, opts *KVTestOptions) {
if testing.Short() {
t.Skip("skipping integration test")
}
if opts == nil {
opts = &KVTestOptions{}
}
if opts.NSPrefix == "" {
opts.NSPrefix = GenerateRandomKVPrefix()
}
t.Logf("Running KV tests with namespace prefix: %s", opts.NSPrefix)
cases := []struct {
name string
fn func(*testing.T, resource.KV, string)
}{
{TestKVGet, runTestKVGet},
{TestKVSave, runTestKVSave},
{TestKVDelete, runTestKVDelete},
{TestKVKeys, runTestKVKeys},
{TestKVKeysWithLimits, runTestKVKeysWithLimits},
{TestKVKeysWithSort, runTestKVKeysWithSort},
{TestKVConcurrent, runTestKVConcurrent},
{TestKVUnixTimestamp, runTestKVUnixTimestamp},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
tc.fn(t, newKV(context.Background()), opts.NSPrefix)
})
}
}
func runTestKVGet(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-get"
t.Run("get existing key", func(t *testing.T) {
// First save a key
testValue := "test value for get"
err := kv.Save(ctx, section, "existing-key", strings.NewReader(testValue))
require.NoError(t, err)
// Now get it
obj, err := kv.Get(ctx, section, "existing-key")
require.NoError(t, err)
assert.Equal(t, "existing-key", obj.Key)
// Read the value
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, testValue, string(value))
// Close the value reader
err = obj.Value.Close()
require.NoError(t, err)
})
t.Run("get non-existent key", func(t *testing.T) {
_, err := kv.Get(ctx, section, "non-existent-key")
assert.Error(t, err)
assert.Equal(t, resource.ErrNotFound, err)
})
t.Run("get with empty section", func(t *testing.T) {
_, err := kv.Get(ctx, "", "some-key")
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
}
func runTestKVSave(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-save"
t.Run("save new key", func(t *testing.T) {
testValue := "new test value"
err := kv.Save(ctx, section, "new-key", strings.NewReader(testValue))
require.NoError(t, err)
// Verify it was saved
obj, err := kv.Get(ctx, section, "new-key")
require.NoError(t, err)
assert.Equal(t, "new-key", obj.Key)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, testValue, string(value))
err = obj.Value.Close()
require.NoError(t, err)
})
t.Run("save overwrite existing key", func(t *testing.T) {
// First save
err := kv.Save(ctx, section, "overwrite-key", strings.NewReader("old value"))
require.NoError(t, err)
// Overwrite
newValue := "new value"
err = kv.Save(ctx, section, "overwrite-key", strings.NewReader(newValue))
require.NoError(t, err)
// Verify it was updated
obj, err := kv.Get(ctx, section, "overwrite-key")
require.NoError(t, err)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, newValue, string(value))
err = obj.Value.Close()
require.NoError(t, err)
})
t.Run("save with empty section", func(t *testing.T) {
err := kv.Save(ctx, "", "some-key", strings.NewReader("some value"))
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
t.Run("save binary data", func(t *testing.T) {
binaryData := []byte{0x00, 0x01, 0x02, 0x03, 0xFF, 0xFE, 0xFD}
err := kv.Save(ctx, section, "binary-key", bytes.NewReader(binaryData))
require.NoError(t, err)
// Verify binary data
obj, err := kv.Get(ctx, section, "binary-key")
require.NoError(t, err)
value, err := io.ReadAll(obj.Value)
require.NoError(t, err)
assert.Equal(t, binaryData, value)
err = obj.Value.Close()
require.NoError(t, err)
})
}
func runTestKVDelete(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-delete"
t.Run("delete existing key", func(t *testing.T) {
// First create a key
err := kv.Save(ctx, section, "delete-key", strings.NewReader("delete me"))
require.NoError(t, err)
// Verify it exists
_, err = kv.Get(ctx, section, "delete-key")
require.NoError(t, err)
// Delete it
err = kv.Delete(ctx, section, "delete-key")
require.NoError(t, err)
// Verify it's gone
_, err = kv.Get(ctx, section, "delete-key")
assert.Error(t, err)
assert.Equal(t, resource.ErrNotFound, err)
})
t.Run("delete non-existent key", func(t *testing.T) {
err := kv.Delete(ctx, section, "non-existent-delete-key")
assert.Error(t, err)
assert.Equal(t, resource.ErrNotFound, err)
})
t.Run("delete with empty section", func(t *testing.T) {
err := kv.Delete(ctx, "", "some-key")
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
}
func runTestKVKeys(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-keys"
// Setup test data
testKeys := []string{"a1", "a2", "b1", "b2", "c1"}
for _, key := range testKeys {
err := kv.Save(ctx, section, key, strings.NewReader("value"+key))
require.NoError(t, err)
}
t.Run("list all keys", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, testKeys, keys)
})
t.Run("list keys with empty section", func(t *testing.T) {
var keys []string
var errors []error
for k, err := range kv.Keys(ctx, "", resource.ListOptions{}) {
if err != nil {
errors = append(errors, err)
break
}
keys = append(keys, k)
}
assert.Len(t, errors, 1)
assert.Contains(t, errors[0].Error(), "section is required")
assert.Empty(t, keys)
})
}
func runTestKVKeysWithLimits(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-keys-limits"
// Setup test data
testKeys := []string{"a1", "a2", "b1", "b2", "c1", "c2", "d1", "d2"}
for _, key := range testKeys {
err := kv.Save(ctx, section, key, strings.NewReader("value"+key))
require.NoError(t, err)
}
t.Run("keys with limit", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{Limit: 3}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a1", "a2", "b1"}, keys)
})
t.Run("keys with range", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{StartKey: "b", EndKey: "d"}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"b1", "b2", "c1", "c2"}, keys)
})
t.Run("keys with prefix", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
StartKey: "c",
EndKey: resource.PrefixRangeEnd("c"),
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"c1", "c2"}, keys)
})
t.Run("keys with limit and range", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
StartKey: "a",
EndKey: "c",
Limit: 2,
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a1", "a2"}, keys)
})
}
func runTestKVKeysWithSort(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-keys-sort"
// Setup test data
testKeys := []string{"a1", "a2", "b1", "b2", "c1"}
for _, key := range testKeys {
err := kv.Save(ctx, section, key, strings.NewReader("value"+key))
require.NoError(t, err)
}
t.Run("keys in ascending order (default)", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{Sort: resource.SortOrderAsc}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a1", "a2", "b1", "b2", "c1"}, keys)
})
t.Run("keys in descending order", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{Sort: resource.SortOrderDesc}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"c1", "b2", "b1", "a2", "a1"}, keys)
})
t.Run("keys descending with prefix", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
StartKey: "a",
EndKey: resource.PrefixRangeEnd("a"),
Sort: resource.SortOrderDesc,
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"a2", "a1"}, keys)
})
t.Run("keys descending with limit", func(t *testing.T) {
var keys []string
for k, err := range kv.Keys(ctx, section, resource.ListOptions{
Sort: resource.SortOrderDesc,
Limit: 3,
}) {
require.NoError(t, err)
keys = append(keys, k)
}
assert.Equal(t, []string{"c1", "b2", "b1"}, keys)
})
}
func runTestKVConcurrent(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(60*time.Second))
section := nsPrefix + "-concurrent"
t.Run("concurrent save and get operations", func(t *testing.T) {
const numGoroutines = 10
const numOperations = 20
done := make(chan error, numGoroutines)
for i := 0; i < numGoroutines; i++ {
go func(goroutineID int) {
var err error
defer func() { done <- err }()
for j := 0; j < numOperations; j++ {
key := fmt.Sprintf("concurrent-key-%d-%d", goroutineID, j)
value := fmt.Sprintf("concurrent-value-%d-%d", goroutineID, j)
// Save
err = kv.Save(ctx, section, key, strings.NewReader(value))
if err != nil {
return
}
// Get immediately
obj, err := kv.Get(ctx, section, key)
if err != nil {
return
}
readValue, err := io.ReadAll(obj.Value)
require.NoError(t, err)
err = obj.Value.Close()
require.NoError(t, err)
assert.Equal(t, value, string(readValue))
}
}(i)
}
// Wait for all goroutines to complete
for i := 0; i < numGoroutines; i++ {
err := <-done
require.NoError(t, err)
}
})
t.Run("concurrent save, delete, and list operations", func(t *testing.T) {
const numGoroutines = 5
done := make(chan error, numGoroutines)
for i := 0; i < numGoroutines; i++ {
go func(goroutineID int) {
var err error
defer func() { done <- err }()
key := fmt.Sprintf("concurrent-ops-key-%d", goroutineID)
value := fmt.Sprintf("concurrent-ops-value-%d", goroutineID)
// Save
err = kv.Save(ctx, section, key, strings.NewReader(value))
if err != nil {
return
}
// List to verify it exists
found := false
for k, err := range kv.Keys(ctx, section, resource.ListOptions{}) {
if err != nil {
return
}
if k == key {
found = true
break
}
}
if !found {
err = fmt.Errorf("key %s not found in list", key)
return
}
// Delete
err = kv.Delete(ctx, section, key)
if err != nil {
return
}
// Verify it's deleted
_, err = kv.Get(ctx, section, key)
require.ErrorIs(t, resource.ErrNotFound, err)
err = nil // Expected error, so clear it
}(i)
}
// Wait for all goroutines to complete
for i := 0; i < numGoroutines; i++ {
err := <-done
require.NoError(t, err)
}
})
}
func runTestKVUnixTimestamp(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
t.Run("unix timestamp returns reasonable value", func(t *testing.T) {
timestamp, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
now := time.Now().Unix()
// Allow for some time difference (up to 5 seconds)
assert.InDelta(t, now, timestamp, 5)
})
t.Run("unix timestamp is consistent", func(t *testing.T) {
timestamp1, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
timestamp2, err := kv.UnixTimestamp(ctx)
require.NoError(t, err)
// Should be very close (within 1 second)
require.InDelta(t, timestamp1, timestamp2, 1)
})
}
pkg/storage/unified/testing/kv_test.go
+28
View File
@@ -0,0 +1,28 @@
package test
import (
"context"
"testing"
badger "github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
func TestBadgerKV(t *testing.T) {
RunKVTest(t, func(ctx context.Context) resource.KV {
opts := badger.DefaultOptions("").WithInMemory(true).WithLogger(nil)
db, err := badger.Open(opts)
require.NoError(t, err)
t.Cleanup(func() {
err := db.Close()
require.NoError(t, err)
})
return resource.NewBadgerKV(db)
}, &KVTestOptions{
NSPrefix: "badger-kv-test",
})
}
pkg/storage/unified/testing/storage_backend.go
+34 -33
View File
@@ -55,10 +55,6 @@ func GenerateRandomNSPrefix() string {
// RunStorageBackendTest runs the storage backend test suite
func RunStorageBackendTest(t *testing.T, newBackend NewBackendFunc, opts *TestOptions) {
if testing.Short() {
t.Skip("skipping integration test")
}
if opts == nil {
opts = &TestOptions{}
}
@@ -228,37 +224,37 @@ func runTestIntegrationBackendGetResourceStats(t *testing.T, backend resource.St
}
// Create resources across different namespaces/groups
_, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED,
WithNamespace(nsPrefix+"-ns1"),
WithNamespace(nsPrefix+"-stats-ns1"),
WithGroup("group"),
WithResource("resource1"))
require.NoError(t, err)
_, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED,
WithNamespace(nsPrefix+"-ns1"),
WithNamespace(nsPrefix+"-stats-ns1"),
WithGroup("group"),
WithResource("resource1"))
require.NoError(t, err)
_, err = writeEvent(ctx, backend, "item3", resourcepb.WatchEvent_ADDED,
WithNamespace(nsPrefix+"-ns1"),
WithNamespace(nsPrefix+"-stats-ns1"),
WithGroup("group"),
WithResource("resource2"))
require.NoError(t, err)
_, err = writeEvent(ctx, backend, "item4", resourcepb.WatchEvent_ADDED,
WithNamespace(nsPrefix+"-ns2"),
WithNamespace(nsPrefix+"-stats-ns2"),
WithGroup("group"),
WithResource("resource1"))
require.NoError(t, err)
_, err = writeEvent(ctx, backend, "item5", resourcepb.WatchEvent_ADDED,
WithNamespace(nsPrefix+"-ns2"),
WithNamespace(nsPrefix+"-stats-ns2"),
WithGroup("group"),
WithResource("resource1"))
require.NoError(t, err)
t.Run("Get stats for ns1", func(t *testing.T) {
stats, err := backend.GetResourceStats(ctx, nsPrefix+"-ns1", 0)
stats, err := backend.GetResourceStats(ctx, nsPrefix+"-stats-ns1", 0)
require.NoError(t, err)
require.Len(t, stats, 2)
@@ -266,14 +262,14 @@ func runTestIntegrationBackendGetResourceStats(t *testing.T, backend resource.St
slices.SortFunc(stats, sortFunc)
// Check first resource stats
require.Equal(t, nsPrefix+"-ns1", stats[0].Namespace)
require.Equal(t, nsPrefix+"-stats-ns1", stats[0].Namespace)
require.Equal(t, "group", stats[0].Group)
require.Equal(t, "resource1", stats[0].Resource)
require.Equal(t, int64(2), stats[0].Count)
require.Greater(t, stats[0].ResourceVersion, int64(0))
// Check second resource stats
require.Equal(t, nsPrefix+"-ns1", stats[1].Namespace)
require.Equal(t, nsPrefix+"-stats-ns1", stats[1].Namespace)
require.Equal(t, "group", stats[1].Group)
require.Equal(t, "resource2", stats[1].Resource)
require.Equal(t, int64(1), stats[1].Count)
@@ -281,11 +277,11 @@ func runTestIntegrationBackendGetResourceStats(t *testing.T, backend resource.St
})
t.Run("Get stats for ns2", func(t *testing.T) {
stats, err := backend.GetResourceStats(ctx, nsPrefix+"-ns2", 0)
stats, err := backend.GetResourceStats(ctx, nsPrefix+"-stats-ns2", 0)
require.NoError(t, err)
require.Len(t, stats, 1)
require.Equal(t, nsPrefix+"-ns2", stats[0].Namespace)
require.Equal(t, nsPrefix+"-stats-ns2", stats[0].Namespace)
require.Equal(t, "group", stats[0].Group)
require.Equal(t, "resource1", stats[0].Resource)
require.Equal(t, int64(2), stats[0].Count)
@@ -293,11 +289,11 @@ func runTestIntegrationBackendGetResourceStats(t *testing.T, backend resource.St
})
t.Run("Get stats with minimum count", func(t *testing.T) {
stats, err := backend.GetResourceStats(ctx, nsPrefix+"-ns1", 1)
stats, err := backend.GetResourceStats(ctx, nsPrefix+"-stats-ns1", 1)
require.NoError(t, err)
require.Len(t, stats, 1)
require.Equal(t, nsPrefix+"-ns1", stats[0].Namespace)
require.Equal(t, nsPrefix+"-stats-ns1", stats[0].Namespace)
require.Equal(t, "group", stats[0].Group)
require.Equal(t, "resource1", stats[0].Resource)
require.Equal(t, int64(2), stats[0].Count)
@@ -314,7 +310,7 @@ func runTestIntegrationBackendWatchWriteEvents(t *testing.T, backend resource.St
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
// Create a few resources before initing the watch
_, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(nsPrefix+"-ns1"))
_, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(nsPrefix+"-watch-ns"))
require.NoError(t, err)
// Start the watch
@@ -322,7 +318,7 @@ func runTestIntegrationBackendWatchWriteEvents(t *testing.T, backend resource.St
require.NoError(t, err)
// Create one more event
_, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(nsPrefix+"-ns1"))
_, err = writeEvent(ctx, backend, "item2", resourcepb.WatchEvent_ADDED, WithNamespace(nsPrefix+"-watch-ns"))
require.NoError(t, err)
require.Equal(t, "item2", (<-stream).Key.Name)
@@ -336,7 +332,7 @@ func runTestIntegrationBackendWatchWriteEvents(t *testing.T, backend resource.St
func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
server := newServer(t, backend)
ns := nsPrefix + "-ns1"
ns := nsPrefix + "-list-ns"
// Create a few resources before starting the watch
rv1, err := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
require.NoError(t, err)
@@ -367,8 +363,9 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
res, err := server.List(ctx, &resourcepb.ListRequest{
Options: &resourcepb.ListOptions{
Key: &resourcepb.ResourceKey{
Group: "group",
Resource: "resource",
Namespace: ns,
Group: "group",
Resource: "resource",
},
},
})
@@ -390,8 +387,9 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
Limit: 3,
Options: &resourcepb.ListOptions{
Key: &resourcepb.ResourceKey{
Group: "group",
Resource: "resource",
Namespace: ns,
Group: "group",
Resource: "resource",
},
},
})
@@ -411,8 +409,9 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
ResourceVersion: rv4,
Options: &resourcepb.ListOptions{
Key: &resourcepb.ResourceKey{
Group: "group",
Resource: "resource",
Namespace: ns,
Group: "group",
Resource: "resource",
},
},
})
@@ -432,8 +431,9 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
ResourceVersion: rv7,
Options: &resourcepb.ListOptions{
Key: &resourcepb.ResourceKey{
Group: "group",
Resource: "resource",
Namespace: ns,
Group: "group",
Resource: "resource",
},
},
})
@@ -462,8 +462,9 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
Limit: 2,
Options: &resourcepb.ListOptions{
Key: &resourcepb.ResourceKey{
Group: "group",
Resource: "resource",
Namespace: ns,
Group: "group",
Resource: "resource",
},
},
})
@@ -484,7 +485,7 @@ func runTestIntegrationBackendList(t *testing.T, backend resource.StorageBackend
func runTestIntegrationBackendListHistory(t *testing.T, backend resource.StorageBackend, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
server := newServer(t, backend)
ns := nsPrefix + "-ns1"
ns := nsPrefix + "-history-ns"
rv1, _ := writeEvent(ctx, backend, "item1", resourcepb.WatchEvent_ADDED, WithNamespace(ns))
require.Greater(t, rv1, int64(0))
@@ -985,12 +986,12 @@ func runTestIntegrationBackendCreateNewResource(t *testing.T, backend resource.S
request := &resourcepb.CreateRequest{
Key: &resourcepb.ResourceKey{
Namespace: "default",
Namespace: ns,
Group: "test.grafana",
Resource: "Test",
Resource: "tests",
Name: "test",
},
Value: []byte(`{"apiVersion":"test.grafana/v0alpha1","kind":"Test","metadata":{"name":"test","namespace":"default"}}`),
Value: []byte(`{"apiVersion":"test.grafana/v0alpha1","kind":"Test","metadata":{"name":"test","namespace":"` + ns + `","uid":"test-uid-123"}}`),
}
response, err := server.Create(ctx, request)
pkg/storage/unified/testing/storage_backend_test.go
+29
View File
@@ -0,0 +1,29 @@
package test
import (
"context"
"testing"
badger "github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
func TestBadgerKVStorageBackend(t *testing.T) {
RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend {
opts := badger.DefaultOptions("").WithInMemory(true).WithLogger(nil)
db, err := badger.Open(opts)
require.NoError(t, err)
t.Cleanup(func() {
_ = db.Close()
})
return resource.NewKvStorageBackend(resource.NewBadgerKV(db))
}, &TestOptions{
NSPrefix: "kvstorage-test",
SkipTests: map[string]bool{
// TODO: fix these tests and remove this skip
TestBlobSupport: true,
},
})
}
pkg/tests/apis/secret/keeper_test.go
-567
View File
@@ -1,567 +0,0 @@
package secret
import (
"context"
"errors"
"math/rand/v2"
"net/http"
"strconv"
"strings"
"testing"
secretv0alpha1 "github.com/grafana/grafana/pkg/apis/secret/v0alpha1"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/tests/apis"
"github.com/grafana/grafana/pkg/tests/testinfra"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
var gvrKeepers = schema.GroupVersionResource{
Group: secretv0alpha1.GROUP,
Version: secretv0alpha1.VERSION,
Resource: secretv0alpha1.KeeperResourceInfo.GetName(),
}
func TestIntegrationKeeper(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for experimental APIs
EnableFeatureToggles: []string{
// Required to start the example service
featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs,
featuremgmt.FlagSecretsManagementAppPlatform,
},
})
permissions := map[string]ResourcePermission{ResourceKeepers: {Actions: ActionsAllKeepers}}
genericUserEditor := mustCreateUsers(t, helper, permissions).Editor
client := helper.GetResourceClient(apis.ResourceClientArgs{
User: genericUserEditor,
GVR: gvrKeepers,
})
t.Run("reading a keeper that does not exist returns a 404", func(t *testing.T) {
raw, err := client.Resource.Get(ctx, "some-keeper-that-does-not-exist", metav1.GetOptions{})
require.Error(t, err)
require.Nil(t, raw)
var statusErr *apierrors.StatusError
require.True(t, errors.As(err, &statusErr))
require.Equal(t, "keeper.secret.grafana.app \"some-keeper-that-does-not-exist\" not found", err.Error())
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
})
t.Run("deleting a keeper that does not exist returns an error", func(t *testing.T) {
err := client.Resource.Delete(ctx, "some-keeper-that-does-not-exist", metav1.DeleteOptions{})
require.Error(t, err)
var statusErr *apierrors.StatusError
require.True(t, errors.As(err, &statusErr))
require.Equal(t, "keeper.secret.grafana.app \"some-keeper-that-does-not-exist\" not found", err.Error())
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
})
t.Run("creating a keeper returns it", func(t *testing.T) {
raw := mustGenerateKeeper(t, helper, genericUserEditor, nil, "testdata/keeper-aws-generate.yaml")
keeper := new(secretv0alpha1.Keeper)
err := runtime.DefaultUnstructuredConverter.FromUnstructured(raw.Object, keeper)
require.NoError(t, err)
require.NotNil(t, keeper)
require.NotEmpty(t, keeper.Spec.Description)
require.NotEmpty(t, keeper.Spec.AWS)
require.Empty(t, keeper.Spec.Azure)
t.Run("and creating another keeper with the same name in the same namespace returns an error", func(t *testing.T) {
testKeeper := helper.LoadYAMLOrJSONFile("testdata/keeper-gcp-generate.yaml")
testKeeper.SetName(raw.GetName())
raw, err := client.Resource.Create(ctx, testKeeper, metav1.CreateOptions{})
require.Error(t, err)
require.Nil(t, raw)
})
t.Run("and reading th keeper returns it same as if when it was created", func(t *testing.T) {
raw, err := client.Resource.Get(ctx, keeper.Name, metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
anotherKeeper := new(secretv0alpha1.Keeper)
err = runtime.DefaultUnstructuredConverter.FromUnstructured(raw.Object, anotherKeeper)
require.NoError(t, err)
require.NotNil(t, anotherKeeper)
require.EqualValues(t, keeper, anotherKeeper)
})
t.Run("and listing keepers returns the created keeper", func(t *testing.T) {
rawList, err := client.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
require.NotNil(t, rawList)
require.GreaterOrEqual(t, len(rawList.Items), 1)
require.Equal(t, keeper.Name, rawList.Items[0].GetName())
})
t.Run("and updating the keeper replaces the spec fields and returns them", func(t *testing.T) {
newRaw := helper.LoadYAMLOrJSONFile("testdata/keeper-gcp-generate.yaml")
newRaw.SetName(raw.GetName())
newRaw.Object["spec"].(map[string]any)["description"] = "New description"
newRaw.Object["metadata"].(map[string]any)["annotations"] = map[string]any{"newAnnotation": "newValue"}
updatedRaw, err := client.Resource.Update(ctx, newRaw, metav1.UpdateOptions{})
require.NoError(t, err)
require.NotNil(t, updatedRaw)
updatedKeeper := new(secretv0alpha1.Keeper)
err = runtime.DefaultUnstructuredConverter.FromUnstructured(updatedRaw.Object, updatedKeeper)
require.NoError(t, err)
require.NotNil(t, updatedKeeper)
require.NotEqualValues(t, updatedKeeper.Spec, keeper.Spec)
})
t.Run("and updating the keeper to reference securevalues that does not exist returns an error", func(t *testing.T) {
newRaw := helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
newRaw.SetName(raw.GetName())
newRaw.Object["spec"].(map[string]any)["aws"] = map[string]any{
"accessKeyId": map[string]any{
"secureValueName": "securevalue-does-not-exist-1",
},
"secretAccessKey": map[string]any{
"secureValueName": "securevalue-does-not-exist-2",
},
}
updatedRaw, err := client.Resource.Update(ctx, newRaw, metav1.UpdateOptions{})
require.Error(t, err)
require.Nil(t, updatedRaw)
require.Contains(t, err.Error(), "securevalue-does-not-exist-1")
require.Contains(t, err.Error(), "securevalue-does-not-exist-2")
})
})
t.Run("creating an invalid keeper fails validation and returns an error", func(t *testing.T) {
testData := helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testData.Object["spec"].(map[string]any)["description"] = ""
raw, err := client.Resource.Create(ctx, testData, metav1.CreateOptions{})
require.Error(t, err)
require.Nil(t, raw)
var statusErr *apierrors.StatusError
require.True(t, errors.As(err, &statusErr))
})
t.Run("creating a keeper with a provider then changing the provider does not return an error", func(t *testing.T) {
rawAWS := mustGenerateKeeper(t, helper, genericUserEditor, nil, "testdata/keeper-aws-generate.yaml")
testDataKeeperGCP := rawAWS.DeepCopy()
testDataKeeperGCP.Object["spec"].(map[string]any)["aws"] = nil
testDataKeeperGCP.Object["spec"].(map[string]any)["gcp"] = map[string]any{
"projectId": "project-id",
"credentialsFile": "/path/to/file.json",
}
rawGCP, err := client.Resource.Update(ctx, testDataKeeperGCP, metav1.UpdateOptions{})
require.NoError(t, err)
require.NotNil(t, rawGCP)
require.NotEqualValues(t, rawAWS.Object["spec"], rawGCP.Object["spec"])
})
t.Run("creating a keeper that references securevalues that does not exist returns an error", func(t *testing.T) {
testDataKeeper := helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testDataKeeper.Object["spec"].(map[string]any)["aws"] = map[string]any{
"accessKeyId": map[string]any{
"secureValueName": "securevalue-does-not-exist-1",
},
"secretAccessKey": map[string]any{
"secureValueName": "securevalue-does-not-exist-2",
},
}
raw, err := client.Resource.Create(ctx, testDataKeeper, metav1.CreateOptions{})
require.Error(t, err)
require.Nil(t, raw)
require.Contains(t, err.Error(), "securevalue-does-not-exist-1")
require.Contains(t, err.Error(), "securevalue-does-not-exist-2")
})
t.Run("deleting a keeper that exists does not return an error", func(t *testing.T) {
generatePrefix := "generated-"
testData := helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testData.SetGenerateName(generatePrefix)
raw, err := client.Resource.Create(ctx, testData, metav1.CreateOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
name := raw.GetName()
require.True(t, strings.HasPrefix(name, generatePrefix))
err = client.Resource.Delete(ctx, name, metav1.DeleteOptions{})
require.NoError(t, err)
t.Run("and then trying to read it returns a 404 error", func(t *testing.T) {
raw, err := client.Resource.Get(ctx, name, metav1.GetOptions{})
require.Error(t, err)
require.Nil(t, raw)
var statusErr *apierrors.StatusError
require.True(t, errors.As(err, &statusErr))
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
})
t.Run("and listing keepers returns an empty list", func(t *testing.T) {
rawList, err := client.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
require.NotNil(t, rawList)
require.Empty(t, rawList.Items)
})
})
t.Run("creating keepers in multiple namespaces", func(t *testing.T) {
permissions := map[string]ResourcePermission{
ResourceKeepers: {Actions: ActionsAllKeepers},
}
editorOrgA := mustCreateUsers(t, helper, permissions).Editor
editorOrgB := mustCreateUsers(t, helper, permissions).Editor
keeperOrgA := mustGenerateKeeper(t, helper, editorOrgA, nil, "testdata/keeper-aws-generate.yaml")
keeperOrgB := mustGenerateKeeper(t, helper, editorOrgB, nil, "testdata/keeper-aws-generate.yaml")
clientOrgA := helper.GetResourceClient(apis.ResourceClientArgs{User: editorOrgA, GVR: gvrKeepers})
clientOrgB := helper.GetResourceClient(apis.ResourceClientArgs{User: editorOrgB, GVR: gvrKeepers})
// Create
t.Run("creating a keeper with the same name as one from another namespace does not return an error", func(t *testing.T) {
// OrgA creating a keeper with the same name from OrgB.
testData := helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testData.SetName(keeperOrgB.GetName())
raw, err := clientOrgA.Resource.Create(ctx, testData, metav1.CreateOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
// OrgA creating a keeper with the same name from OrgB.
testData = helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testData.SetName(keeperOrgA.GetName())
raw, err = clientOrgB.Resource.Create(ctx, testData, metav1.CreateOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
require.NoError(t, clientOrgA.Resource.Delete(ctx, keeperOrgB.GetName(), metav1.DeleteOptions{}))
require.NoError(t, clientOrgB.Resource.Delete(ctx, keeperOrgA.GetName(), metav1.DeleteOptions{}))
})
// Read
t.Run("fetching a keeper from another namespace returns not found", func(t *testing.T) {
var statusErr *apierrors.StatusError
// OrgA trying to fetch keeper from OrgB.
raw, err := clientOrgA.Resource.Get(ctx, keeperOrgB.GetName(), metav1.GetOptions{})
require.Error(t, err)
require.Nil(t, raw)
require.True(t, errors.As(err, &statusErr))
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
// OrgB trying to fetch keeper from OrgA.
raw, err = clientOrgB.Resource.Get(ctx, keeperOrgA.GetName(), metav1.GetOptions{})
require.Error(t, err)
require.Nil(t, raw)
require.True(t, errors.As(err, &statusErr))
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
})
// Update
t.Run("updating a keeper from another namespace returns not found", func(t *testing.T) {
var statusErr *apierrors.StatusError
// OrgA trying to update securevalue from OrgB.
testData := helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testData.SetName(keeperOrgB.GetName())
testData.Object["spec"].(map[string]any)["description"] = "New description"
raw, err := clientOrgA.Resource.Update(ctx, testData, metav1.UpdateOptions{})
require.Error(t, err)
require.Nil(t, raw)
require.True(t, errors.As(err, &statusErr))
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
// OrgB trying to update keeper from OrgA.
testData = helper.LoadYAMLOrJSONFile("testdata/keeper-aws-generate.yaml")
testData.SetName(keeperOrgA.GetName())
testData.Object["spec"].(map[string]any)["description"] = "New description"
raw, err = clientOrgB.Resource.Update(ctx, testData, metav1.UpdateOptions{})
require.Error(t, err)
require.Nil(t, raw)
require.True(t, errors.As(err, &statusErr))
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
})
// Delete
t.Run("deleting a keeper from another namespace returns an error and does not delete it", func(t *testing.T) {
var statusErr *apierrors.StatusError
// OrgA trying to delete keeper from OrgB.
err := clientOrgA.Resource.Delete(ctx, keeperOrgB.GetName(), metav1.DeleteOptions{})
require.Error(t, err)
require.True(t, errors.As(err, &statusErr))
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
// Check that it still exists from the perspective of OrgB.
raw, err := clientOrgB.Resource.Get(ctx, keeperOrgB.GetName(), metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
// OrgB trying to delete keeper from OrgA.
err = clientOrgB.Resource.Delete(ctx, keeperOrgA.GetName(), metav1.DeleteOptions{})
require.Error(t, err)
require.Equal(t, http.StatusNotFound, int(statusErr.Status().Code))
// Check that it still exists from the perspective of OrgA.
raw, err = clientOrgA.Resource.Get(ctx, keeperOrgA.GetName(), metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
})
// List
t.Run("listing keeper from a namespace does not return the ones from another namespace", func(t *testing.T) {
// OrgA listing keeper.
listOrgA, err := clientOrgA.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
require.NotNil(t, listOrgA)
require.Len(t, listOrgA.Items, 1)
require.Equal(t, *keeperOrgA, listOrgA.Items[0])
// OrgB listing keeper.
listOrgB, err := clientOrgB.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
require.NotNil(t, listOrgB)
require.Len(t, listOrgB.Items, 1)
require.Equal(t, *keeperOrgB, listOrgB.Items[0])
})
})
t.Run("keeper actions without having required permissions", func(t *testing.T) {
// Create users on a random org without specifying secrets-related permissions.
editorP := mustCreateUsers(t, helper, nil).Editor
clientP := helper.GetResourceClient(apis.ResourceClientArgs{
User: editorP,
GVR: gvrKeepers,
})
// GET
rawGet, err := clientP.Resource.Get(ctx, "some-keeper", metav1.GetOptions{})
require.Error(t, err)
require.Nil(t, rawGet)
var statusGetErr *apierrors.StatusError
require.True(t, errors.As(err, &statusGetErr))
require.EqualValues(t, http.StatusForbidden, statusGetErr.Status().Code)
// LIST
rawList, err := clientP.Resource.List(ctx, metav1.ListOptions{})
require.Error(t, err)
require.Nil(t, rawList)
var statusListErr *apierrors.StatusError
require.True(t, errors.As(err, &statusListErr))
require.EqualValues(t, http.StatusForbidden, statusListErr.Status().Code)
// CREATE
testKeeper := helper.LoadYAMLOrJSONFile("testdata/keeper-gcp-generate.yaml") // to pass validation before authz.
rawCreate, err := clientP.Resource.Create(ctx, testKeeper, metav1.CreateOptions{})
require.Error(t, err)
require.Nil(t, rawCreate)
var statusCreateErr *apierrors.StatusError
require.True(t, errors.As(err, &statusCreateErr))
require.EqualValues(t, http.StatusForbidden, statusCreateErr.Status().Code)
// UPDATE
testKeeper.SetName("test") // to pass validation before authz.
rawUpdate, err := clientP.Resource.Update(ctx, testKeeper, metav1.UpdateOptions{})
require.Error(t, err)
require.Nil(t, rawUpdate)
var statusUpdateErr *apierrors.StatusError
require.True(t, errors.As(err, &statusUpdateErr))
require.EqualValues(t, http.StatusForbidden, statusUpdateErr.Status().Code)
// DELETE
err = clientP.Resource.Delete(ctx, "some-keeper", metav1.DeleteOptions{})
require.Error(t, err)
var statusDeleteErr *apierrors.StatusError
require.True(t, errors.As(err, &statusDeleteErr))
require.EqualValues(t, http.StatusForbidden, statusDeleteErr.Status().Code)
})
t.Run("keeper actions with permissions but with limited scope", func(t *testing.T) {
suffix := strconv.FormatInt(rand.Int64(), 10)
// Fix the Keeper names.
keeperName := "kp-" + suffix
testKeeper := helper.LoadYAMLOrJSONFile("testdata/keeper-gcp-generate.yaml")
testKeeper.SetName(keeperName)
keeperNameAnother := "kp-another-" + suffix
testKeeperAnother := helper.LoadYAMLOrJSONFile("testdata/keeper-gcp-generate.yaml")
testKeeperAnother.SetName(keeperNameAnother)
// Fix the org ID because we will create another user with scope "all" permissions on the same org, to compare.
orgID := rand.Int64() + 2
// Permissions which allow any action, but scoped actions (get, update, delete) only on `keeperName` and NO OTHER Keeper.
scopedLimitedPermissions := map[string]ResourcePermission{
ResourceKeepers: {
Actions: ActionsAllKeepers,
Name: keeperName,
},
}
// Create users (+ client) with permission to manage ONLY the Keeper `keeperName`.
editorLimited := mustCreateUsersWithOrg(t, helper, orgID, scopedLimitedPermissions).Editor
clientScopedLimited := helper.GetResourceClient(apis.ResourceClientArgs{
User: editorLimited,
GVR: gvrKeepers,
})
// Create users (+ client) with permission to manage ANY Keepers.
scopedAllPermissions := map[string]ResourcePermission{
ResourceKeepers: {
Actions: ActionsAllKeepers,
Name: "*", // this or not sending a `Name` have the same effect.
},
}
editorAll := mustCreateUsersWithOrg(t, helper, orgID, scopedAllPermissions).Editor
clientScopedAll := helper.GetResourceClient(apis.ResourceClientArgs{
User: editorAll,
GVR: gvrKeepers,
})
// For create, we don't have actual granular permissions, so we can use any client that has unscoped create permissions.
// This is because when we don't know yet what the name of the resource will be, the request comes with an empty value.
// And thus the authorizer can't do granular checks.
t.Run("CREATE", func(t *testing.T) {
rawCreateLimited, err := clientScopedAll.Resource.Create(ctx, testKeeper, metav1.CreateOptions{})
require.NoError(t, err)
require.NotNil(t, rawCreateLimited)
rawCreateLimited, err = clientScopedAll.Resource.Create(ctx, testKeeperAnother, metav1.CreateOptions{})
require.NoError(t, err)
require.NotNil(t, rawCreateLimited)
})
t.Run("READ", func(t *testing.T) {
// Retrieve `keeperName` from the limited client.
rawGetLimited, err := clientScopedLimited.Resource.Get(ctx, keeperName, metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, rawGetLimited)
require.Equal(t, rawGetLimited.GetUID(), rawGetLimited.GetUID())
// Retrieve `keeperName` from the scope-all client.
rawGetAll, err := clientScopedAll.Resource.Get(ctx, keeperName, metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, rawGetAll)
require.Equal(t, rawGetAll.GetUID(), rawGetLimited.GetUID())
// Even though we can create it, we cannot retrieve `keeperNameAnother` from the limited client.
rawGetLimited, err = clientScopedLimited.Resource.Get(ctx, keeperNameAnother, metav1.GetOptions{})
require.Error(t, err)
require.Nil(t, rawGetLimited)
var statusGetErr *apierrors.StatusError
require.True(t, errors.As(err, &statusGetErr))
require.EqualValues(t, http.StatusForbidden, statusGetErr.Status().Code)
// Retrieve `keeperNameAnother` from the scope-all client.
rawGetAll, err = clientScopedAll.Resource.Get(ctx, keeperNameAnother, metav1.GetOptions{})
require.NoError(t, err)
require.NotNil(t, rawGetAll)
})
t.Run("LIST", func(t *testing.T) {
// List Keepers from the limited client should return only 1.
rawList, err := clientScopedLimited.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
require.NotNil(t, rawList)
require.Len(t, rawList.Items, 1)
require.Equal(t, keeperName, rawList.Items[0].GetName())
// List Keepers from the scope-all client should return all of them.
rawList, err = clientScopedAll.Resource.List(ctx, metav1.ListOptions{})
require.NoError(t, err)
require.NotNil(t, rawList)
require.Len(t, rawList.Items, 2)
})
t.Run("UPDATE", func(t *testing.T) {
// Update `keeperName` from the limited client.
testKeeperUpdate := testKeeper.DeepCopy()
testKeeperUpdate.Object["spec"].(map[string]any)["description"] = "keeper-description-1234"
rawUpdate, err := clientScopedLimited.Resource.Update(ctx, testKeeperUpdate, metav1.UpdateOptions{})
require.NoError(t, err)
require.NotNil(t, rawUpdate)
// Try to update `keeperNameAnother` from the limited client.
testKeeperAnotherUpdate := testKeeperAnother.DeepCopy()
testKeeperAnotherUpdate.Object["spec"].(map[string]any)["description"] = "keeper-description-5678"
rawUpdate, err = clientScopedLimited.Resource.Update(ctx, testKeeperAnotherUpdate, metav1.UpdateOptions{})
require.Error(t, err)
require.Nil(t, rawUpdate)
var statusUpdateErr *apierrors.StatusError
require.True(t, errors.As(err, &statusUpdateErr))
require.EqualValues(t, http.StatusForbidden, statusUpdateErr.Status().Code)
// Update `keeperNameAnother` from the scope-all client.
rawUpdate, err = clientScopedAll.Resource.Update(ctx, testKeeperAnotherUpdate, metav1.UpdateOptions{})
require.NoError(t, err)
require.NotNil(t, rawUpdate)
})
// Keep this last for cleaning up the resources.
t.Run("DELETE", func(t *testing.T) {
// Try to delete `keeperNameAnother` from the limited client.
err := clientScopedLimited.Resource.Delete(ctx, keeperNameAnother, metav1.DeleteOptions{})
require.Error(t, err)
var statusDeleteErr *apierrors.StatusError
require.True(t, errors.As(err, &statusDeleteErr))
require.EqualValues(t, http.StatusForbidden, statusDeleteErr.Status().Code)
// Delete `keeperNameAnother` from the scope-all client.
err = clientScopedAll.Resource.Delete(ctx, keeperNameAnother, metav1.DeleteOptions{})
require.NoError(t, err)
// Delete `keeperName` from the limited client.
err = clientScopedLimited.Resource.Delete(ctx, keeperName, metav1.DeleteOptions{})
require.NoError(t, err)
})
})
}
pkg/tests/apis/secret/main_test.go
-150
View File
@@ -1,150 +0,0 @@
package secret
import (
"cmp"
"context"
"encoding/json"
"math/rand/v2"
"strconv"
"testing"
"github.com/grafana/grafana/pkg/registry/apis/secret"
"github.com/grafana/grafana/pkg/services/accesscontrol/resourcepermissions"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/team"
"github.com/grafana/grafana/pkg/tests/apis"
"github.com/grafana/grafana/pkg/tests/testinfra"
"github.com/grafana/grafana/pkg/tests/testsuite"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
var (
ResourceSecureValues = "secret.securevalues"
ResourceKeepers = "secret.keepers"
ActionsAllKeepers = []string{
secret.ActionSecretKeepersCreate,
secret.ActionSecretKeepersWrite,
secret.ActionSecretKeepersRead,
secret.ActionSecretKeepersDelete,
}
ActionsAllSecureValues = []string{
secret.ActionSecretSecureValuesCreate,
secret.ActionSecretSecureValuesWrite,
secret.ActionSecretSecureValuesRead,
secret.ActionSecretSecureValuesDelete,
}
)
type ResourcePermission struct {
Actions []string
Name string // empty or "*" for all
}
func TestMain(m *testing.M) {
testsuite.Run(m)
}
func TestIntegrationDiscoveryClient(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
AppModeProduction: false, // required for experimental APIs
EnableFeatureToggles: []string{
// Required to start the example service
featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs,
featuremgmt.FlagSecretsManagementAppPlatform,
},
})
t.Run("check discovery client", func(t *testing.T) {
disco := helper.NewDiscoveryClient()
resources, err := disco.ServerResourcesForGroupVersion("secret.grafana.app/v0alpha1")
require.NoError(t, err)
v1Disco, err := json.MarshalIndent(resources, "", " ")
require.NoError(t, err)
var apiResourceList map[string]any
require.NoError(t, json.Unmarshal(v1Disco, &apiResourceList))
groupVersion, ok := apiResourceList["groupVersion"].(string)
require.True(t, ok)
require.Equal(t, "secret.grafana.app/v0alpha1", groupVersion)
apiResources, ok := apiResourceList["resources"].([]any)
require.True(t, ok)
require.Len(t, apiResources, 2) // securevalue + keeper + (subresources...)
})
}
func mustCreateUsersWithOrg(t *testing.T, helper *apis.K8sTestHelper, orgID int64, permissionMap map[string]ResourcePermission) apis.OrgUsers {
t.Helper()
permissions := make([]resourcepermissions.SetResourcePermissionCommand, 0, len(permissionMap))
for resource, permission := range permissionMap {
permissions = append(permissions, resourcepermissions.SetResourcePermissionCommand{
Actions: permission.Actions,
Resource: resource,
ResourceAttribute: "uid",
ResourceID: cmp.Or(permission.Name, "*"),
})
}
orgName := "org-" + strconv.FormatInt(orgID, 10)
userSuffix := strconv.FormatInt(rand.Int64(), 10)
// Add here admin or viewer if necessary.
editor := helper.CreateUser("editor-"+userSuffix, orgName, org.RoleEditor, permissions)
staff := helper.CreateTeam("staff-"+userSuffix, "staff-"+userSuffix+"@"+orgName, editor.Identity.GetOrgID())
// Also call this method for each new user.
helper.AddOrUpdateTeamMember(editor, staff.ID, team.PermissionTypeMember)
return apis.OrgUsers{
Editor: editor,
Staff: staff,
}
}
func mustCreateUsers(t *testing.T, helper *apis.K8sTestHelper, permissionMap map[string]ResourcePermission) apis.OrgUsers {
orgID := rand.Int64() + 2 // if it is 0, becomes 2 and not 1.
return mustCreateUsersWithOrg(t, helper, orgID, permissionMap)
}
func mustGenerateKeeper(t *testing.T, helper *apis.K8sTestHelper, user apis.User, specType map[string]any, testFile string) *unstructured.Unstructured {
t.Helper()
require.NotEmpty(t, testFile, "testFile must not be empty")
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
keeperClient := helper.GetResourceClient(apis.ResourceClientArgs{
User: user,
GVR: gvrKeepers,
})
testKeeper := helper.LoadYAMLOrJSONFile(testFile)
if specType != nil {
testKeeper.Object["spec"] = specType
}
raw, err := keeperClient.Resource.Create(ctx, testKeeper, metav1.CreateOptions{})
require.NoError(t, err)
require.NotNil(t, raw)
t.Cleanup(func() {
require.NoError(t, keeperClient.Resource.Delete(ctx, raw.GetName(), metav1.DeleteOptions{}))
})
return raw
}
pkg/tests/apis/secret/testdata/keeper-aws-generate.yaml
Vendored
-15
View File
@@ -1,15 +0,0 @@
apiVersion: secret.grafana.app/v0alpha1
kind: Keeper
metadata:
annotations:
xx: XXX
labels:
aa: AAA
spec:
description: AWS XYZ value
aws:
accessKeyId:
valueFromEnv: ACCESS_KEY_ID_XYZ
secretAccessKey:
valueFromEnv: SECRET_ACCESS_KEY_XYZ
kmsKeyId: kmsKeyId-xyz
pkg/tests/apis/secret/testdata/keeper-gcp-generate.yaml
Vendored
-14
View File
@@ -1,14 +0,0 @@
apiVersion: secret.grafana.app/v0alpha1
kind: Keeper
metadata:
annotations:
xx: XXX
yy: YYY
labels:
aa: AAA
bb: BBB
spec:
description: GCP XYZ value
gcp:
projectId: project-id
credentialsFile: /path/to/file.json
pkg/tests/apis/secret/testdata/secure-value-default-generate.yaml
Vendored
-15
View File
@@ -1,15 +0,0 @@
apiVersion: secret.grafana.app/v0alpha1
kind: SecureValue
metadata:
annotations:
xx: XXX
yy: YYY
labels:
aa: AAA
bb: BBB
spec:
description: This is a secret
value: this is super duper secure
decrypters:
- k6
- synthetic-monitoring
pkg/tests/apis/secret/testdata/secure-value-generate.yaml
Vendored
-16
View File
@@ -1,16 +0,0 @@
apiVersion: secret.grafana.app/v0alpha1
kind: SecureValue
metadata:
annotations:
xx: XXX
yy: YYY
labels:
aa: AAA
bb: BBB
spec:
description: XYZ value
keeper: my-keeper-1
value: super duper secure
decrypters:
- k6
- synthetic-monitoring
public/app/core/components/ForgottenPassword/ChangePasswordPage.test.tsx
+1 -1
View File
@@ -1,7 +1,7 @@
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { getRouteComponentProps } from 'app/core/navigation/__mocks__/routeProps';
import { getRouteComponentProps } from 'app/core/navigation/mocks/routeProps';
import { ChangePasswordPage, Props } from './ChangePasswordPage';
public/app/core/components/Signup/SignupPage.test.tsx
+1 -1
View File
@@ -1,6 +1,6 @@
import { render, fireEvent, screen, waitFor, userEvent } from 'test/test-utils';
import { getRouteComponentProps } from 'app/core/navigation/__mocks__/routeProps';
import { getRouteComponentProps } from 'app/core/navigation/mocks/routeProps';
import { SignupPage } from './SignupPage';
public/app/core/components/TimelineChart/utils.ts
+6 -4
View File
@@ -327,7 +327,7 @@ export function prepareTimelineFields(
for (let i = 0; i < frame.fields.length; i++) {
let f = frame.fields[i];
if (f.type === FieldType.time) {
if (f.type === FieldType.time && typeof f.values[0] === 'number') {
if (startFieldIdx === -1) {
startFieldIdx = i;
} else if (endFieldIdx === -1) {
@@ -390,9 +390,11 @@ export function prepareTimelineFields(
for (let field of frame.fields) {
switch (field.type) {
case FieldType.time:
isTimeseries = true;
hasTimeseries = true;
fields.push(field);
if (typeof field.values[0] === 'number') {
isTimeseries = true;
hasTimeseries = true;
fields.push(field);
}
break;
case FieldType.enum:
case FieldType.number:
public/app/core/navigation/__mocks__/routeProps.ts → public/app/core/navigation/mocks/routeProps.ts
View File
public/app/core/services/FetchQueue.test.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { take } from 'rxjs/operators';
import { BackendSrvRequest } from '@grafana/runtime';
import { FetchQueue, FetchQueueUpdate, FetchStatus } from './FetchQueue';
import { subscribeTester } from './__mocks__/subscribeTester';
import { subscribeTester } from './mocks/subscribeTester';
describe('FetchQueue', () => {
describe('add', () => {
public/app/core/services/ResponseQueue.test.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import { BackendSrvRequest } from '@grafana/runtime';
import { FetchQueue, FetchQueueUpdate } from './FetchQueue';
import { ResponseQueue } from './ResponseQueue';
import { subscribeTester } from './__mocks__/subscribeTester';
import { subscribeTester } from './mocks/subscribeTester';
const getTestContext = () => {
const id = 'id';
public/app/core/services/__mocks__/search_srv.ts → public/app/core/services/mocks/search_srv.ts
View File
public/app/core/services/__mocks__/subscribeTester.ts → public/app/core/services/mocks/subscribeTester.ts
View File
public/app/core/utils/shortLinks.test.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { LogRowModel } from '@grafana/data';
import { config } from '@grafana/runtime';
import { createLogRow } from 'app/features/logs/components/__mocks__/logRow';
import { createLogRow } from 'app/features/logs/components/mocks/logRow';
import { createShortLink, createAndCopyShortLink, getLogsPermalinkRange } from './shortLinks';
public/app/features/admin/Users/OrgUsersTable.test.tsx
+1 -1
View File
@@ -2,7 +2,7 @@ import { render, screen } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { OrgUser } from '../../../types';
import { getMockUsers } from '../../users/__mocks__/userMocks';
import { getMockUsers } from '../../users/mocks/userMocks';
import { OrgUsersTable, Props } from './OrgUsersTable';
public/app/features/alerting/unified/Receivers.test.tsx
+1 -1
View File
@@ -6,7 +6,7 @@ import {
PROVISIONED_MIMIR_ALERTMANAGER_UID,
mockDataSources,
setupVanillaAlertmanagerServer,
} from 'app/features/alerting/unified/components/settings/__mocks__/server';
} from 'app/features/alerting/unified/components/settings/mocks/server';
import { setupMswServer } from 'app/features/alerting/unified/mockApi';
import { grantUserPermissions } from 'app/features/alerting/unified/mocks';
import { setupDataSources } from 'app/features/alerting/unified/testSetup/datasources';
public/app/features/alerting/unified/Settings.test.tsx
+2 -2
View File
@@ -3,8 +3,8 @@ import { render } from 'test/test-utils';
import { byRole, byTestId, byText } from 'testing-library-selector';
import SettingsPage from './Settings';
import DataSourcesResponse from './components/settings/__mocks__/api/datasources.json';
import { setupGrafanaManagedServer, withExternalOnlySetting } from './components/settings/__mocks__/server';
import DataSourcesResponse from './components/settings/mocks/api/datasources.json';
import { setupGrafanaManagedServer, withExternalOnlySetting } from './components/settings/mocks/server';
import { setupMswServer } from './mockApi';
import { grantUserRole } from './mocks';
public/app/features/alerting/unified/components/contact-points/ContactPoints.test.tsx
+3 -3
View File
@@ -17,11 +17,11 @@ import { DataSourceType, GRAFANA_RULES_SOURCE_NAME } from '../../utils/datasourc
import { ContactPoint } from './ContactPoint';
import { ContactPointsPageContents } from './ContactPoints';
import setupMimirFlavoredServer from './__mocks__/mimirFlavoredServer';
import { RECEIVER_META_KEY } from './constants';
import setupMimirFlavoredServer from './mocks/mimirFlavoredServer';
import setupVanillaAlertmanagerFlavoredServer, {
VANILLA_ALERTMANAGER_DATASOURCE_UID,
} from './__mocks__/vanillaAlertmanagerServer';
import { RECEIVER_META_KEY } from './constants';
} from './mocks/vanillaAlertmanagerServer';
import { ContactPointWithMetadata, ReceiverConfigWithMetadata, RouteReference } from './utils';
/**
public/app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.mimir.config.mock.json → public/app/features/alerting/unified/components/contact-points/mocks/alertmanager.mimir.config.mock.json
View File
public/app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.vanilla.mock.json → public/app/features/alerting/unified/components/contact-points/mocks/alertmanager.vanilla.mock.json
View File
public/app/features/alerting/unified/components/contact-points/__mocks__/mimirFlavoredServer.ts → public/app/features/alerting/unified/components/contact-points/mocks/mimirFlavoredServer.ts
View File
public/app/features/alerting/unified/components/contact-points/__mocks__/receivers.mock.json → public/app/features/alerting/unified/components/contact-points/mocks/receivers.mock.json
View File
public/app/features/alerting/unified/components/contact-points/__mocks__/vanillaAlertmanagerServer.ts → public/app/features/alerting/unified/components/contact-points/mocks/vanillaAlertmanagerServer.ts
View File
public/app/features/alerting/unified/components/settings/AlertmanagerConfig.test.tsx
+1 -1
View File
@@ -16,7 +16,7 @@ import {
PROVISIONED_MIMIR_ALERTMANAGER_UID,
mockDataSources,
setupVanillaAlertmanagerServer,
} from './__mocks__/server';
} from './mocks/server';
const renderConfiguration = (
alertManagerSourceName: string,
public/app/features/alerting/unified/components/settings/__mocks__/api/alertmanager/grafana/config/api/v1/alerts.json → public/app/features/alerting/unified/components/settings/mocks/api/alertmanager/grafana/config/api/v1/alerts.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/api/alertmanager/grafana/config/history.json → public/app/features/alerting/unified/components/settings/mocks/api/alertmanager/grafana/config/history.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/api/alertmanager/provisioned/config/api/v1/alerts.json → public/app/features/alerting/unified/components/settings/mocks/api/alertmanager/provisioned/config/api/v1/alerts.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/api/alertmanager/vanilla prometheus/api/v2/status.json → public/app/features/alerting/unified/components/settings/mocks/api/alertmanager/vanilla prometheus/api/v2/status.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/api/datasources.json → public/app/features/alerting/unified/components/settings/mocks/api/datasources.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/api/v1/ngalert/admin_config.json → public/app/features/alerting/unified/components/settings/mocks/api/v1/ngalert/admin_config.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/api/v1/ngalert/alertmanagers.json → public/app/features/alerting/unified/components/settings/mocks/api/v1/ngalert/alertmanagers.json
View File
public/app/features/alerting/unified/components/settings/__mocks__/server.ts → public/app/features/alerting/unified/components/settings/mocks/server.ts
View File
public/app/features/alerting/unified/mocks/server/handlers/alertmanagers.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import { HttpResponse, JsonBodyType, StrictResponse, http } from 'msw';
import { TemplatesTestPayload } from 'app/features/alerting/unified/api/templateApi';
import receiversMock from 'app/features/alerting/unified/components/contact-points/__mocks__/receivers.mock.json';
import receiversMock from 'app/features/alerting/unified/components/contact-points/mocks/receivers.mock.json';
import { MOCK_SILENCE_ID_EXISTING, mockAlertmanagerAlert } from 'app/features/alerting/unified/mocks';
import { defaultGrafanaAlertingConfigurationStatusResponse } from 'app/features/alerting/unified/mocks/alertmanagerApi';
import {
public/app/features/alerting/unified/state/AlertingQueryRunner.test.ts
+1 -1
View File
@@ -20,7 +20,7 @@ import { BackendSrv } from 'app/core/services/backend_srv';
import {
EXTERNAL_VANILLA_ALERTMANAGER_UID,
mockDataSources,
} from 'app/features/alerting/unified/components/settings/__mocks__/server';
} from 'app/features/alerting/unified/components/settings/mocks/server';
import { setupMswServer } from 'app/features/alerting/unified/mockApi';
import { setupDataSources } from 'app/features/alerting/unified/testSetup/datasources';
import { ExpressionQuery, ExpressionQueryType } from 'app/features/expressions/types';
public/app/features/annotations/executeAnnotationQuery.test.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { DataSourceApi, dateTime, DataQuery } from '@grafana/data';
import { PanelModel } from '../dashboard/state/PanelModel';
import { createDashboardModelFixture } from '../dashboard/state/__fixtures__/dashboardFixtures';
import { TestQuery, getMockDataSource } from '../query/state/__mocks__/mockDataSource';
import { TestQuery, getMockDataSource } from '../query/state/mocks/mockDataSource';
import { executeAnnotationQuery } from './executeAnnotationQuery';
import { AnnotationQueryOptions } from './types';

Some files were not shown because too many files have changed in this diff Show More