diff --git a/.github/actions/change-detection/action.yml b/.github/actions/change-detection/action.yml index 2b8484d7cf6..1fd10ae5540 100644 --- a/.github/actions/change-detection/action.yml +++ b/.github/actions/change-detection/action.yml @@ -14,6 +14,9 @@ outputs: frontend: description: Whether the frontend or self has changed in any way value: ${{ steps.changed-files.outputs.frontend_any_changed || 'true' }} + frontend-packages: + description: Whether any frontend packages have changed + value: ${{ steps.changed-files.outputs.frontend_packages_any_changed || 'true' }} e2e: description: Whether the e2e tests or self have changed in any way value: ${{ steps.changed-files.outputs.e2e_any_changed == 'true' || @@ -97,6 +100,12 @@ runs: - '.yarn/**' - 'apps/dashboard/pkg/migration/**' - '${{ inputs.self }}' + frontend_packages: + - '.github/actions/checkout/**' + - '.github/actions/change-detection/**' + - 'packages/**' + - './scripts/validate-npm-packages.sh' + - '${{ inputs.self }}' e2e: - 'e2e/**' - 'e2e-playwright/**' @@ -153,6 +162,8 @@ runs: echo " --> ${{ steps.changed-files.outputs.backend_all_changed_files }}" echo "Frontend: ${{ steps.changed-files.outputs.frontend_any_changed || 'true' }}" echo " --> ${{ steps.changed-files.outputs.frontend_all_changed_files }}" + echo "Frontend packages: ${{ steps.changed-files.outputs.frontend_packages_any_changed || 'true' }}" + echo " --> ${{ steps.changed-files.outputs.frontend_packages_all_changed_files }}" echo "E2E: ${{ steps.changed-files.outputs.e2e_any_changed || 'true' }}" echo " --> ${{ steps.changed-files.outputs.e2e_all_changed_files }}" echo " --> ${{ steps.changed-files.outputs.backend_all_changed_files }}" diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml index 5762389f83b..92ffd43593c 100644 --- a/.github/actions/setup-node/action.yml +++ b/.github/actions/setup-node/action.yml @@ -4,8 +4,8 @@ description: Sets up a node.js environment with presets for the Grafana reposito runs: using: "composite" steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version-file: '.nvmrc' cache: 'yarn' - cache-dependency-path: 'yarn.lock' \ No newline at end of file + cache-dependency-path: 'yarn.lock' diff --git a/.github/workflows/frontend-lint.yml b/.github/workflows/frontend-lint.yml index 02833d61149..539d57b1742 100644 --- a/.github/workflows/frontend-lint.yml +++ b/.github/workflows/frontend-lint.yml @@ -17,6 +17,7 @@ jobs: outputs: changed: ${{ steps.detect-changes.outputs.frontend }} prettier: ${{ steps.detect-changes.outputs.frontend == 'true' || steps.detect-changes.outputs.docs == 'true' }} + changed-frontend-packages: ${{ steps.detect-changes.outputs.frontend-packages }} steps: - uses: actions/checkout@v5 with: @@ -42,11 +43,8 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: 'yarn' - cache-dependency-path: 'yarn.lock' + - name: Setup Node + uses: ./.github/actions/setup-node - run: yarn install --immutable --check-cache - run: yarn run prettier:check - run: yarn run lint @@ -63,11 +61,8 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: 'yarn' - cache-dependency-path: 'yarn.lock' + - name: Setup Node + uses: ./.github/actions/setup-node - name: Setup Enterprise uses: ./.github/actions/setup-enterprise with: @@ -89,11 +84,8 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: 'yarn' - cache-dependency-path: 'yarn.lock' + - name: Setup Node + uses: ./.github/actions/setup-node - run: yarn install --immutable --check-cache - run: yarn run typecheck lint-frontend-typecheck-enterprise: @@ -109,11 +101,8 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: 'yarn' - cache-dependency-path: 'yarn.lock' + - name: Setup Node + uses: ./.github/actions/setup-node - name: Setup Enterprise uses: ./.github/actions/setup-enterprise with: @@ -133,11 +122,8 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: 'yarn' - cache-dependency-path: 'yarn.lock' + - name: Setup Node + uses: ./.github/actions/setup-node - run: yarn install --immutable --check-cache - name: Generate API clients run: | @@ -164,11 +150,8 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: 'yarn' - cache-dependency-path: 'yarn.lock' + - name: Setup Node + uses: ./.github/actions/setup-node - name: Setup Enterprise uses: ./.github/actions/setup-enterprise with: @@ -187,3 +170,26 @@ jobs: echo "${uncommited_error_message}" exit 1 fi + lint-frontend-packed-packages: + needs: detect-changes + permissions: + contents: read + id-token: write + if: github.event_name == 'pull_request' && needs.detect-changes.outputs.changed-frontend-packages == 'true' + name: Verify packed frontend packages + runs-on: ubuntu-latest + steps: + - name: Checkout build commit + uses: actions/checkout@v5 + with: + persist-credentials: false + - name: Setup Node + uses: ./.github/actions/setup-node + - name: Install dependencies + run: yarn install --immutable + - name: Build and pack packages + run: | + yarn run packages:build + yarn run packages:pack + - name: Validate packages + run: ./scripts/validate-npm-packages.sh diff --git a/apps/dashboard/pkg/migration/conversion/testdata/input/v2alpha1.ds-data-query.json b/apps/dashboard/pkg/migration/conversion/testdata/input/v2alpha1.ds-data-query.json index a27c33c3239..ac88b83370f 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/input/v2alpha1.ds-data-query.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/input/v2alpha1.ds-data-query.json @@ -852,6 +852,194 @@ } } } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Single Dashboard DS Query", + "description": "Panel with a single -- Dashboard -- datasource query", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "stat", + "spec": { + "pluginVersion": "12.1.0-pre", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Multiple Dashboard DS Queries", + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 2, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 3, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "stat", + "spec": { + "pluginVersion": "12.1.0-pre", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -914,6 +1102,24 @@ "name": "panel-6" } } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } } ] } diff --git a/apps/dashboard/pkg/migration/conversion/testdata/input/v2beta1.ds-data-query.json b/apps/dashboard/pkg/migration/conversion/testdata/input/v2beta1.ds-data-query.json index 0c787609714..fad72787d19 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/input/v2beta1.ds-data-query.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/input/v2beta1.ds-data-query.json @@ -879,6 +879,200 @@ } } } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Single Dashboard DS Query", + "description": "Panel with a single -- Dashboard -- datasource query", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.1.0-pre", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Multiple Dashboard DS Queries", + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 2, + "withTransforms": true + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 3, + "withTransforms": true + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.1.0-pre", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -973,6 +1167,32 @@ "name": "panel-6" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 6, + "width": 8, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 6, + "width": 8, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } } ] } diff --git a/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v0alpha1.json b/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v0alpha1.json index 20f70a0a647..b38cf688949 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v0alpha1.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v0alpha1.json @@ -711,6 +711,146 @@ ], "title": "Mixed DS WITHOUT REFS", "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "description": "Panel with a single -- Dashboard -- datasource query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + } + ], + "title": "Single Dashboard DS Query", + "type": "stat" + }, + { + "datasource": { + "type": "mixed", + "uid": "-- Mixed --" + }, + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "B", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 3, + "refId": "C", + "withTransforms": true + } + ], + "title": "Multiple Dashboard DS Queries", + "type": "stat" } ], "preload": false, diff --git a/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v1beta1.json b/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v1beta1.json index 9956ad6962f..0b7f512e6f1 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v1beta1.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v1beta1.json @@ -711,6 +711,146 @@ ], "title": "Mixed DS WITHOUT REFS", "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "description": "Panel with a single -- Dashboard -- datasource query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + } + ], + "title": "Single Dashboard DS Query", + "type": "stat" + }, + { + "datasource": { + "type": "mixed", + "uid": "-- Mixed --" + }, + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "B", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 3, + "refId": "C", + "withTransforms": true + } + ], + "title": "Multiple Dashboard DS Queries", + "type": "stat" } ], "preload": false, diff --git a/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v2beta1.json b/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v2beta1.json index 09e35c64258..aba5db6146d 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v2beta1.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/output/v2alpha1.ds-data-query.v2beta1.json @@ -879,6 +879,200 @@ } } } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Single Dashboard DS Query", + "description": "Panel with a single -- Dashboard -- datasource query", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.1.0-pre", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Multiple Dashboard DS Queries", + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 2, + "withTransforms": true + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "datasource", + "version": "v0", + "datasource": { + "name": "-- Dashboard --" + }, + "spec": { + "panelId": 3, + "withTransforms": true + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.1.0-pre", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -941,6 +1135,24 @@ "name": "panel-6" } } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } } ] } diff --git a/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v0alpha1.json b/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v0alpha1.json index 4494023eb13..99bcc3e9581 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v0alpha1.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v0alpha1.json @@ -711,6 +711,146 @@ ], "title": "Mixed DS WITHOUT REFS", "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "description": "Panel with a single -- Dashboard -- datasource query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + } + ], + "title": "Single Dashboard DS Query", + "type": "stat" + }, + { + "datasource": { + "type": "mixed", + "uid": "-- Mixed --" + }, + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "B", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 3, + "refId": "C", + "withTransforms": true + } + ], + "title": "Multiple Dashboard DS Queries", + "type": "stat" } ], "preload": false, diff --git a/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v1beta1.json b/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v1beta1.json index bc8d90d796a..e2d54185ea5 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v1beta1.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v1beta1.json @@ -711,6 +711,146 @@ ], "title": "Mixed DS WITHOUT REFS", "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "description": "Panel with a single -- Dashboard -- datasource query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + } + ], + "title": "Single Dashboard DS Query", + "type": "stat" + }, + { + "datasource": { + "type": "mixed", + "uid": "-- Mixed --" + }, + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0-pre", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 2, + "refId": "B", + "withTransforms": true + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 3, + "refId": "C", + "withTransforms": true + } + ], + "title": "Multiple Dashboard DS Queries", + "type": "stat" } ], "preload": false, diff --git a/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v2alpha1.json b/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v2alpha1.json index bb70e99ec48..d3ca201e380 100644 --- a/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v2alpha1.json +++ b/apps/dashboard/pkg/migration/conversion/testdata/output/v2beta1.ds-data-query.v2alpha1.json @@ -852,6 +852,194 @@ } } } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "id": 7, + "title": "Single Dashboard DS Query", + "description": "Panel with a single -- Dashboard -- datasource query", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "stat", + "spec": { + "pluginVersion": "12.1.0-pre", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "id": 8, + "title": "Multiple Dashboard DS Queries", + "description": "Panel with multiple -- Dashboard -- datasource queries (should be mixed)", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 1, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 2, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "datasource", + "spec": { + "panelId": 3, + "withTransforms": true + } + }, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "stat", + "spec": { + "pluginVersion": "12.1.0-pre", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -946,6 +1134,32 @@ "name": "panel-6" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 6, + "width": 8, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-7" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 6, + "width": 8, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-8" + } + } } ] } diff --git a/apps/dashboard/pkg/migration/conversion/v2alpha1_to_v1beta1.go b/apps/dashboard/pkg/migration/conversion/v2alpha1_to_v1beta1.go index 46a2a533d41..5dc7ecf21fd 100644 --- a/apps/dashboard/pkg/migration/conversion/v2alpha1_to_v1beta1.go +++ b/apps/dashboard/pkg/migration/conversion/v2alpha1_to_v1beta1.go @@ -1195,16 +1195,36 @@ func getDataSourceForQuery(explicitDS *dashv2alpha1.DashboardDataSourceRef, quer // getPanelDatasource determines the panel-level datasource for V1. // Returns: // - Mixed datasource reference if queries use different datasources +// - Mixed datasource reference if multiple queries use Dashboard datasource (they fetch from different panels) +// - Dashboard datasource reference if a single query uses Dashboard datasource // - First query's datasource if all queries use the same datasource // - nil if no queries exist // Compares based on V2 input without runtime resolution: // - If query has explicit datasource.uid โ†’ use that UID and type // - Else โ†’ use query.Kind as type (empty UID) func getPanelDatasource(queries []dashv2alpha1.DashboardPanelQueryKind) map[string]interface{} { + const sharedDashboardQuery = "-- Dashboard --" + if len(queries) == 0 { return nil } + // Count how many queries use Dashboard datasource + // Multiple dashboard queries need mixed mode because they fetch from different panels + // which may have different underlying datasources + dashboardDsQueryCount := 0 + for _, query := range queries { + if query.Spec.Datasource != nil && query.Spec.Datasource.Uid != nil && *query.Spec.Datasource.Uid == sharedDashboardQuery { + dashboardDsQueryCount++ + } + } + if dashboardDsQueryCount > 1 { + return map[string]interface{}{ + "type": "mixed", + "uid": "-- Mixed --", + } + } + var firstUID, firstType string var hasFirst bool @@ -1239,6 +1259,16 @@ func getPanelDatasource(queries []dashv2alpha1.DashboardPanelQueryKind) map[stri } } + // Handle case when a single query uses Dashboard datasource. + // This is needed for the frontend to properly activate and fetch data from source panels. + // See DashboardDatasourceBehaviour.tsx for more details. + if firstUID == sharedDashboardQuery { + return map[string]interface{}{ + "type": "datasource", + "uid": sharedDashboardQuery, + } + } + // Not mixed - return the first query's datasource so the panel has a datasource set. // This is required because the frontend's legacy PanelModel.PanelQueryRunner.run uses panel.datasource // to resolve the datasource, and if undefined, it falls back to the default datasource diff --git a/apps/provisioning/pkg/apis/provisioning/v0alpha1/connections.go b/apps/provisioning/pkg/apis/provisioning/v0alpha1/connections.go index f9f8dcc8382..2738af49db1 100644 --- a/apps/provisioning/pkg/apis/provisioning/v0alpha1/connections.go +++ b/apps/provisioning/pkg/apis/provisioning/v0alpha1/connections.go @@ -32,7 +32,7 @@ type ConnectionSecure struct { // Token is the reference of the token used to act as the Connection. // This value is stored securely and cannot be read back - Token common.InlineSecureValue `json:"webhook,omitzero,omitempty"` + Token common.InlineSecureValue `json:"token,omitzero,omitempty"` } func (v ConnectionSecure) IsZero() bool { diff --git a/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go b/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go index 11788142e94..4db11489c98 100644 --- a/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go +++ b/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi.go @@ -320,7 +320,7 @@ func schema_pkg_apis_provisioning_v0alpha1_ConnectionSecure(ref common.Reference Ref: ref("github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1.InlineSecureValue"), }, }, - "webhook": { + "token": { SchemaProps: spec.SchemaProps{ Description: "Token is the reference of the token used to act as the Connection. This value is stored securely and cannot be read back", Default: map[string]interface{}{}, diff --git a/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list b/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list index 3a54dcf2a5e..72567e04b90 100644 --- a/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list +++ b/apps/provisioning/pkg/apis/provisioning/v0alpha1/zz_generated.openapi_violation_exceptions.list @@ -22,7 +22,6 @@ API rule violation: list_type_missing,github.com/grafana/grafana/apps/provisioni API rule violation: list_type_missing,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,ResourceList,Items API rule violation: list_type_missing,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,TestResults,Errors API rule violation: list_type_missing,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,WebhookStatus,SubscribedEvents -API rule violation: names_match,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,ConnectionSecure,Token API rule violation: names_match,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,ConnectionSpec,GitHub API rule violation: names_match,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,JobSpec,PullRequest API rule violation: names_match,github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1,JobStatus,URLs diff --git a/apps/provisioning/pkg/connection/connection.go b/apps/provisioning/pkg/connection/connection.go new file mode 100644 index 00000000000..d2043b5af4a --- /dev/null +++ b/apps/provisioning/pkg/connection/connection.go @@ -0,0 +1,16 @@ +package connection + +import ( + "context" +) + +//go:generate mockery --name Connection --structname MockConnection --inpackage --filename connection_mock.go --with-expecter +type Connection interface { + // Validate ensures the resource _looks_ correct. + // It should be called before trying to upsert a resource into the Kubernetes API server. + // This is not an indication that the connection information works, just that they are reasonably configured. + Validate(ctx context.Context) error + + // Mutate performs in place mutation of the underneath resource. + Mutate(context.Context) error +} diff --git a/apps/provisioning/pkg/connection/connection_mock.go b/apps/provisioning/pkg/connection/connection_mock.go new file mode 100644 index 00000000000..3867059d432 --- /dev/null +++ b/apps/provisioning/pkg/connection/connection_mock.go @@ -0,0 +1,128 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package connection + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// MockConnection is an autogenerated mock type for the Connection type +type MockConnection struct { + mock.Mock +} + +type MockConnection_Expecter struct { + mock *mock.Mock +} + +func (_m *MockConnection) EXPECT() *MockConnection_Expecter { + return &MockConnection_Expecter{mock: &_m.Mock} +} + +// Mutate provides a mock function with given fields: _a0 +func (_m *MockConnection) Mutate(_a0 context.Context) error { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Mutate") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockConnection_Mutate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Mutate' +type MockConnection_Mutate_Call struct { + *mock.Call +} + +// Mutate is a helper method to define mock.On call +// - _a0 context.Context +func (_e *MockConnection_Expecter) Mutate(_a0 interface{}) *MockConnection_Mutate_Call { + return &MockConnection_Mutate_Call{Call: _e.mock.On("Mutate", _a0)} +} + +func (_c *MockConnection_Mutate_Call) Run(run func(_a0 context.Context)) *MockConnection_Mutate_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockConnection_Mutate_Call) Return(_a0 error) *MockConnection_Mutate_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockConnection_Mutate_Call) RunAndReturn(run func(context.Context) error) *MockConnection_Mutate_Call { + _c.Call.Return(run) + return _c +} + +// Validate provides a mock function with given fields: ctx +func (_m *MockConnection) Validate(ctx context.Context) error { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for Validate") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(ctx) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockConnection_Validate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Validate' +type MockConnection_Validate_Call struct { + *mock.Call +} + +// Validate is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockConnection_Expecter) Validate(ctx interface{}) *MockConnection_Validate_Call { + return &MockConnection_Validate_Call{Call: _e.mock.On("Validate", ctx)} +} + +func (_c *MockConnection_Validate_Call) Run(run func(ctx context.Context)) *MockConnection_Validate_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockConnection_Validate_Call) Return(_a0 error) *MockConnection_Validate_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockConnection_Validate_Call) RunAndReturn(run func(context.Context) error) *MockConnection_Validate_Call { + _c.Call.Return(run) + return _c +} + +// NewMockConnection creates a new instance of MockConnection. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockConnection(t interface { + mock.TestingT + Cleanup(func()) +}) *MockConnection { + mock := &MockConnection{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/apps/provisioning/pkg/connection/extra_mock.go b/apps/provisioning/pkg/connection/extra_mock.go new file mode 100644 index 00000000000..cc2a1f3d5e2 --- /dev/null +++ b/apps/provisioning/pkg/connection/extra_mock.go @@ -0,0 +1,141 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package connection + +import ( + context "context" + + v0alpha1 "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + mock "github.com/stretchr/testify/mock" +) + +// MockExtra is an autogenerated mock type for the Extra type +type MockExtra struct { + mock.Mock +} + +type MockExtra_Expecter struct { + mock *mock.Mock +} + +func (_m *MockExtra) EXPECT() *MockExtra_Expecter { + return &MockExtra_Expecter{mock: &_m.Mock} +} + +// Build provides a mock function with given fields: ctx, r +func (_m *MockExtra) Build(ctx context.Context, r *v0alpha1.Connection) (Connection, error) { + ret := _m.Called(ctx, r) + + if len(ret) == 0 { + panic("no return value specified for Build") + } + + var r0 Connection + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Connection) (Connection, error)); ok { + return rf(ctx, r) + } + if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Connection) Connection); ok { + r0 = rf(ctx, r) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(Connection) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *v0alpha1.Connection) error); ok { + r1 = rf(ctx, r) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockExtra_Build_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Build' +type MockExtra_Build_Call struct { + *mock.Call +} + +// Build is a helper method to define mock.On call +// - ctx context.Context +// - r *v0alpha1.Connection +func (_e *MockExtra_Expecter) Build(ctx interface{}, r interface{}) *MockExtra_Build_Call { + return &MockExtra_Build_Call{Call: _e.mock.On("Build", ctx, r)} +} + +func (_c *MockExtra_Build_Call) Run(run func(ctx context.Context, r *v0alpha1.Connection)) *MockExtra_Build_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*v0alpha1.Connection)) + }) + return _c +} + +func (_c *MockExtra_Build_Call) Return(_a0 Connection, _a1 error) *MockExtra_Build_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockExtra_Build_Call) RunAndReturn(run func(context.Context, *v0alpha1.Connection) (Connection, error)) *MockExtra_Build_Call { + _c.Call.Return(run) + return _c +} + +// Type provides a mock function with no fields +func (_m *MockExtra) Type() v0alpha1.ConnectionType { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Type") + } + + var r0 v0alpha1.ConnectionType + if rf, ok := ret.Get(0).(func() v0alpha1.ConnectionType); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(v0alpha1.ConnectionType) + } + + return r0 +} + +// MockExtra_Type_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Type' +type MockExtra_Type_Call struct { + *mock.Call +} + +// Type is a helper method to define mock.On call +func (_e *MockExtra_Expecter) Type() *MockExtra_Type_Call { + return &MockExtra_Type_Call{Call: _e.mock.On("Type")} +} + +func (_c *MockExtra_Type_Call) Run(run func()) *MockExtra_Type_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockExtra_Type_Call) Return(_a0 v0alpha1.ConnectionType) *MockExtra_Type_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockExtra_Type_Call) RunAndReturn(run func() v0alpha1.ConnectionType) *MockExtra_Type_Call { + _c.Call.Return(run) + return _c +} + +// NewMockExtra creates a new instance of MockExtra. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockExtra(t interface { + mock.TestingT + Cleanup(func()) +}) *MockExtra { + mock := &MockExtra{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/apps/provisioning/pkg/connection/factory.go b/apps/provisioning/pkg/connection/factory.go new file mode 100644 index 00000000000..4a0e46d84d2 --- /dev/null +++ b/apps/provisioning/pkg/connection/factory.go @@ -0,0 +1,75 @@ +package connection + +import ( + "context" + "fmt" + "sort" + + provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" +) + +//go:generate mockery --name=Extra --structname=MockExtra --inpackage --filename=extra_mock.go --with-expecter +type Extra interface { + Type() provisioning.ConnectionType + Build(ctx context.Context, r *provisioning.Connection) (Connection, error) +} + +//go:generate mockery --name=Factory --structname=MockFactory --inpackage --filename=factory_mock.go --with-expecter +type Factory interface { + Types() []provisioning.ConnectionType + Build(ctx context.Context, r *provisioning.Connection) (Connection, error) +} + +type factory struct { + extras map[provisioning.ConnectionType]Extra + enabled map[provisioning.ConnectionType]struct{} +} + +func ProvideFactory(enabled map[provisioning.ConnectionType]struct{}, extras []Extra) (Factory, error) { + f := &factory{ + enabled: enabled, + extras: make(map[provisioning.ConnectionType]Extra, len(extras)), + } + + for _, e := range extras { + if _, exists := f.extras[e.Type()]; exists { + return nil, fmt.Errorf("connection type %q is already registered", e.Type()) + } + f.extras[e.Type()] = e + } + + return f, nil +} + +func (f *factory) Types() []provisioning.ConnectionType { + var types []provisioning.ConnectionType + for t := range f.enabled { + if _, exists := f.extras[t]; exists { + types = append(types, t) + } + } + + sort.Slice(types, func(i, j int) bool { + return string(types[i]) < string(types[j]) + }) + + return types +} + +func (f *factory) Build(ctx context.Context, c *provisioning.Connection) (Connection, error) { + for _, e := range f.extras { + if e.Type() == c.Spec.Type { + if _, enabled := f.enabled[e.Type()]; !enabled { + return nil, fmt.Errorf("connection type %q is not enabled", e.Type()) + } + + return e.Build(ctx, c) + } + } + + return nil, fmt.Errorf("connection type %q is not supported", c.Spec.Type) +} + +var ( + _ Factory = (*factory)(nil) +) diff --git a/apps/provisioning/pkg/connection/factory_mock.go b/apps/provisioning/pkg/connection/factory_mock.go new file mode 100644 index 00000000000..8fd7023920f --- /dev/null +++ b/apps/provisioning/pkg/connection/factory_mock.go @@ -0,0 +1,143 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package connection + +import ( + context "context" + + v0alpha1 "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + mock "github.com/stretchr/testify/mock" +) + +// MockFactory is an autogenerated mock type for the Factory type +type MockFactory struct { + mock.Mock +} + +type MockFactory_Expecter struct { + mock *mock.Mock +} + +func (_m *MockFactory) EXPECT() *MockFactory_Expecter { + return &MockFactory_Expecter{mock: &_m.Mock} +} + +// Build provides a mock function with given fields: ctx, r +func (_m *MockFactory) Build(ctx context.Context, r *v0alpha1.Connection) (Connection, error) { + ret := _m.Called(ctx, r) + + if len(ret) == 0 { + panic("no return value specified for Build") + } + + var r0 Connection + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Connection) (Connection, error)); ok { + return rf(ctx, r) + } + if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Connection) Connection); ok { + r0 = rf(ctx, r) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(Connection) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *v0alpha1.Connection) error); ok { + r1 = rf(ctx, r) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockFactory_Build_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Build' +type MockFactory_Build_Call struct { + *mock.Call +} + +// Build is a helper method to define mock.On call +// - ctx context.Context +// - r *v0alpha1.Connection +func (_e *MockFactory_Expecter) Build(ctx interface{}, r interface{}) *MockFactory_Build_Call { + return &MockFactory_Build_Call{Call: _e.mock.On("Build", ctx, r)} +} + +func (_c *MockFactory_Build_Call) Run(run func(ctx context.Context, r *v0alpha1.Connection)) *MockFactory_Build_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*v0alpha1.Connection)) + }) + return _c +} + +func (_c *MockFactory_Build_Call) Return(_a0 Connection, _a1 error) *MockFactory_Build_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockFactory_Build_Call) RunAndReturn(run func(context.Context, *v0alpha1.Connection) (Connection, error)) *MockFactory_Build_Call { + _c.Call.Return(run) + return _c +} + +// Types provides a mock function with no fields +func (_m *MockFactory) Types() []v0alpha1.ConnectionType { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Types") + } + + var r0 []v0alpha1.ConnectionType + if rf, ok := ret.Get(0).(func() []v0alpha1.ConnectionType); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]v0alpha1.ConnectionType) + } + } + + return r0 +} + +// MockFactory_Types_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Types' +type MockFactory_Types_Call struct { + *mock.Call +} + +// Types is a helper method to define mock.On call +func (_e *MockFactory_Expecter) Types() *MockFactory_Types_Call { + return &MockFactory_Types_Call{Call: _e.mock.On("Types")} +} + +func (_c *MockFactory_Types_Call) Run(run func()) *MockFactory_Types_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockFactory_Types_Call) Return(_a0 []v0alpha1.ConnectionType) *MockFactory_Types_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockFactory_Types_Call) RunAndReturn(run func() []v0alpha1.ConnectionType) *MockFactory_Types_Call { + _c.Call.Return(run) + return _c +} + +// NewMockFactory creates a new instance of MockFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockFactory(t interface { + mock.TestingT + Cleanup(func()) +}) *MockFactory { + mock := &MockFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/apps/provisioning/pkg/connection/factory_test.go b/apps/provisioning/pkg/connection/factory_test.go new file mode 100644 index 00000000000..4ce6bc96e0e --- /dev/null +++ b/apps/provisioning/pkg/connection/factory_test.go @@ -0,0 +1,309 @@ +package connection + +import ( + "context" + "errors" + "testing" + + provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestProvideFactory(t *testing.T) { + t.Run("should create factory with valid extras", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + extra2 := NewMockExtra(t) + extra2.EXPECT().Type().Return(provisioning.GitlabConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + provisioning.GitlabConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1, extra2}) + require.NoError(t, err) + require.NotNil(t, factory) + }) + + t.Run("should create factory with empty extras", func(t *testing.T) { + enabled := map[provisioning.ConnectionType]struct{}{} + + factory, err := ProvideFactory(enabled, []Extra{}) + require.NoError(t, err) + require.NotNil(t, factory) + }) + + t.Run("should create factory with nil enabled map", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + factory, err := ProvideFactory(nil, []Extra{extra1}) + require.NoError(t, err) + require.NotNil(t, factory) + }) + + t.Run("should return error when duplicate repository types", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + extra2 := NewMockExtra(t) + extra2.EXPECT().Type().Return(provisioning.GithubConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1, extra2}) + require.Error(t, err) + assert.Nil(t, factory) + assert.Contains(t, err.Error(), "connection type \"github\" is already registered") + }) +} + +func TestFactory_Types(t *testing.T) { + t.Run("should return only enabled types that have extras", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + extra2 := NewMockExtra(t) + extra2.EXPECT().Type().Return(provisioning.GitlabConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + provisioning.GitlabConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1, extra2}) + require.NoError(t, err) + + types := factory.Types() + assert.Len(t, types, 2) + assert.Contains(t, types, provisioning.GithubConnectionType) + assert.Contains(t, types, provisioning.GitlabConnectionType) + }) + + t.Run("should return sorted list of types", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GitlabConnectionType) + + extra2 := NewMockExtra(t) + extra2.EXPECT().Type().Return(provisioning.GithubConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + provisioning.GitlabConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1, extra2}) + require.NoError(t, err) + + types := factory.Types() + assert.Len(t, types, 2) + // github should come before gitlab alphabetically + assert.Equal(t, provisioning.GithubConnectionType, types[0]) + assert.Equal(t, provisioning.GitlabConnectionType, types[1]) + }) + + t.Run("should return empty list when no types are enabled", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{} + + factory, err := ProvideFactory(enabled, []Extra{extra1}) + require.NoError(t, err) + + types := factory.Types() + assert.Empty(t, types) + }) + + t.Run("should not return types that are enabled but have no extras", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + provisioning.GitlabConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1}) + require.NoError(t, err) + + types := factory.Types() + assert.Len(t, types, 1) + assert.Contains(t, types, provisioning.GithubConnectionType) + assert.NotContains(t, types, provisioning.GitlabConnectionType) + }) + + t.Run("should not return types that have extras but are not enabled", func(t *testing.T) { + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + extra2 := NewMockExtra(t) + extra2.EXPECT().Type().Return(provisioning.GitlabConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1, extra2}) + require.NoError(t, err) + + types := factory.Types() + assert.Len(t, types, 1) + assert.Contains(t, types, provisioning.GithubConnectionType) + assert.NotContains(t, types, provisioning.GitlabConnectionType) + }) + + t.Run("should return empty list when no extras are provided", func(t *testing.T) { + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{}) + require.NoError(t, err) + + types := factory.Types() + assert.Empty(t, types) + }) +} + +func TestFactory_Build(t *testing.T) { + t.Run("should successfully build connection when type is enabled and has extra", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + }, + } + + mockConnection := NewMockConnection(t) + extra := NewMockExtra(t) + extra.EXPECT().Type().Return(provisioning.GithubConnectionType) + extra.EXPECT().Build(ctx, conn).Return(mockConnection, nil) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra}) + require.NoError(t, err) + + result, err := factory.Build(ctx, conn) + require.NoError(t, err) + assert.Equal(t, mockConnection, result) + }) + + t.Run("should return error when type is not enabled", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GitlabConnectionType, + }, + } + + extra := NewMockExtra(t) + extra.EXPECT().Type().Return(provisioning.GitlabConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra}) + require.NoError(t, err) + + result, err := factory.Build(ctx, conn) + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "connection type \"gitlab\" is not enabled") + }) + + t.Run("should return error when type is not supported", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GitlabConnectionType, + }, + } + + extra := NewMockExtra(t) + extra.EXPECT().Type().Return(provisioning.GithubConnectionType) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra}) + require.NoError(t, err) + + result, err := factory.Build(ctx, conn) + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "connection type \"gitlab\" is not supported") + }) + + t.Run("should pass through errors from extra.Build()", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + }, + } + + expectedErr := errors.New("build error") + extra := NewMockExtra(t) + extra.EXPECT().Type().Return(provisioning.GithubConnectionType) + extra.EXPECT().Build(ctx, conn).Return(nil, expectedErr) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra}) + require.NoError(t, err) + + result, err := factory.Build(ctx, conn) + require.Error(t, err) + assert.Nil(t, result) + assert.Equal(t, expectedErr, err) + }) + + t.Run("should build with multiple extras registered", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GitlabConnectionType, + }, + } + + mockConnection := NewMockConnection(t) + + extra1 := NewMockExtra(t) + extra1.EXPECT().Type().Return(provisioning.GithubConnectionType) + + extra2 := NewMockExtra(t) + extra2.EXPECT().Type().Return(provisioning.GitlabConnectionType) + extra2.EXPECT().Build(ctx, conn).Return(mockConnection, nil) + + enabled := map[provisioning.ConnectionType]struct{}{ + provisioning.GithubConnectionType: {}, + provisioning.GitlabConnectionType: {}, + } + + factory, err := ProvideFactory(enabled, []Extra{extra1, extra2}) + require.NoError(t, err) + + result, err := factory.Build(ctx, conn) + require.NoError(t, err) + assert.Equal(t, mockConnection, result) + }) +} diff --git a/apps/provisioning/pkg/connection/github/client.go b/apps/provisioning/pkg/connection/github/client.go new file mode 100644 index 00000000000..7ddb9a4665e --- /dev/null +++ b/apps/provisioning/pkg/connection/github/client.go @@ -0,0 +1,93 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/google/go-github/v70/github" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// API errors that we need to convey after parsing real GH errors (or faking them). +var ( + //lint:ignore ST1005 this is not punctuation + ErrServiceUnavailable = apierrors.NewServiceUnavailable("github is unavailable") +) + +//go:generate mockery --name Client --structname MockClient --inpackage --filename client_mock.go --with-expecter +type Client interface { + // Apps and installations + GetApp(ctx context.Context) (App, error) + GetAppInstallation(ctx context.Context, installationID string) (AppInstallation, error) +} + +// App represents a Github App. +type App struct { + // ID represents the GH app ID. + ID int64 + // Slug represents the GH app slug. + Slug string + // Owner represents the GH account/org owning the app + Owner string +} + +// AppInstallation represents a Github App Installation. +type AppInstallation struct { + // ID represents the GH installation ID. + ID int64 + // Whether the installation is enabled or not. + Enabled bool +} + +type githubClient struct { + gh *github.Client +} + +func NewClient(client *github.Client) Client { + return &githubClient{client} +} + +// GetApp gets the app by using the given token. +func (r *githubClient) GetApp(ctx context.Context) (App, error) { + app, _, err := r.gh.Apps.Get(ctx, "") + if err != nil { + var ghErr *github.ErrorResponse + if errors.As(err, &ghErr) && ghErr.Response.StatusCode == http.StatusServiceUnavailable { + return App{}, ErrServiceUnavailable + } + return App{}, err + } + + // TODO(ferruvich): do we need any other info? + return App{ + ID: app.GetID(), + Slug: app.GetSlug(), + Owner: app.GetOwner().GetLogin(), + }, nil +} + +// GetAppInstallation gets the installation of the app related to the given token. +func (r *githubClient) GetAppInstallation(ctx context.Context, installationID string) (AppInstallation, error) { + id, err := strconv.Atoi(installationID) + if err != nil { + return AppInstallation{}, fmt.Errorf("invalid installation ID: %s", installationID) + } + + installation, _, err := r.gh.Apps.GetInstallation(ctx, int64(id)) + if err != nil { + var ghErr *github.ErrorResponse + if errors.As(err, &ghErr) && ghErr.Response.StatusCode == http.StatusServiceUnavailable { + return AppInstallation{}, ErrServiceUnavailable + } + return AppInstallation{}, err + } + + // TODO(ferruvich): do we need any other info? + return AppInstallation{ + ID: installation.GetID(), + Enabled: installation.GetSuspendedAt().IsZero(), + }, nil +} diff --git a/apps/provisioning/pkg/connection/github/client_mock.go b/apps/provisioning/pkg/connection/github/client_mock.go new file mode 100644 index 00000000000..c9f009f5021 --- /dev/null +++ b/apps/provisioning/pkg/connection/github/client_mock.go @@ -0,0 +1,149 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package github + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// MockClient is an autogenerated mock type for the Client type +type MockClient struct { + mock.Mock +} + +type MockClient_Expecter struct { + mock *mock.Mock +} + +func (_m *MockClient) EXPECT() *MockClient_Expecter { + return &MockClient_Expecter{mock: &_m.Mock} +} + +// GetApp provides a mock function with given fields: ctx +func (_m *MockClient) GetApp(ctx context.Context) (App, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetApp") + } + + var r0 App + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (App, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) App); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(App) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockClient_GetApp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetApp' +type MockClient_GetApp_Call struct { + *mock.Call +} + +// GetApp is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockClient_Expecter) GetApp(ctx interface{}) *MockClient_GetApp_Call { + return &MockClient_GetApp_Call{Call: _e.mock.On("GetApp", ctx)} +} + +func (_c *MockClient_GetApp_Call) Run(run func(ctx context.Context)) *MockClient_GetApp_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockClient_GetApp_Call) Return(_a0 App, _a1 error) *MockClient_GetApp_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockClient_GetApp_Call) RunAndReturn(run func(context.Context) (App, error)) *MockClient_GetApp_Call { + _c.Call.Return(run) + return _c +} + +// GetAppInstallation provides a mock function with given fields: ctx, installationID +func (_m *MockClient) GetAppInstallation(ctx context.Context, installationID string) (AppInstallation, error) { + ret := _m.Called(ctx, installationID) + + if len(ret) == 0 { + panic("no return value specified for GetAppInstallation") + } + + var r0 AppInstallation + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (AppInstallation, error)); ok { + return rf(ctx, installationID) + } + if rf, ok := ret.Get(0).(func(context.Context, string) AppInstallation); ok { + r0 = rf(ctx, installationID) + } else { + r0 = ret.Get(0).(AppInstallation) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, installationID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockClient_GetAppInstallation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAppInstallation' +type MockClient_GetAppInstallation_Call struct { + *mock.Call +} + +// GetAppInstallation is a helper method to define mock.On call +// - ctx context.Context +// - installationID string +func (_e *MockClient_Expecter) GetAppInstallation(ctx interface{}, installationID interface{}) *MockClient_GetAppInstallation_Call { + return &MockClient_GetAppInstallation_Call{Call: _e.mock.On("GetAppInstallation", ctx, installationID)} +} + +func (_c *MockClient_GetAppInstallation_Call) Run(run func(ctx context.Context, installationID string)) *MockClient_GetAppInstallation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MockClient_GetAppInstallation_Call) Return(_a0 AppInstallation, _a1 error) *MockClient_GetAppInstallation_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockClient_GetAppInstallation_Call) RunAndReturn(run func(context.Context, string) (AppInstallation, error)) *MockClient_GetAppInstallation_Call { + _c.Call.Return(run) + return _c +} + +// NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockClient(t interface { + mock.TestingT + Cleanup(func()) +}) *MockClient { + mock := &MockClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/apps/provisioning/pkg/connection/github/client_test.go b/apps/provisioning/pkg/connection/github/client_test.go new file mode 100644 index 00000000000..bae6d6ac1e9 --- /dev/null +++ b/apps/provisioning/pkg/connection/github/client_test.go @@ -0,0 +1,297 @@ +package github_test + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/google/go-github/v70/github" + conngh "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" + mockhub "github.com/migueleliasweb/go-github-mock/src/mock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGithubClient_GetApp(t *testing.T) { + tests := []struct { + name string + mockHandler *http.Client + token string + wantApp conngh.App + wantErr error + }{ + { + name: "get app successfully", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetApp, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + app := &github.App{ + ID: github.Ptr(int64(12345)), + Slug: github.Ptr("my-test-app"), + Owner: &github.User{ + Login: github.Ptr("grafana"), + }, + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(app)) + }), + ), + ), + token: "test-token", + wantApp: conngh.App{ + ID: 12345, + Slug: "my-test-app", + Owner: "grafana", + }, + wantErr: nil, + }, + { + name: "service unavailable", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetApp, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusServiceUnavailable, + }, + Message: "Service unavailable", + })) + }), + ), + ), + token: "test-token", + wantApp: conngh.App{}, + wantErr: conngh.ErrServiceUnavailable, + }, + { + name: "other error", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetApp, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusInternalServerError, + }, + Message: "Internal server error", + })) + }), + ), + ), + token: "test-token", + wantApp: conngh.App{}, + wantErr: &github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusInternalServerError, + }, + Message: "Internal server error", + }, + }, + { + name: "unauthorized error", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetApp, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusUnauthorized, + }, + Message: "Bad credentials", + })) + }), + ), + ), + token: "invalid-token", + wantApp: conngh.App{}, + wantErr: &github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusUnauthorized, + }, + Message: "Bad credentials", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a mock client + ghClient := github.NewClient(tt.mockHandler) + client := conngh.NewClient(ghClient) + + // Call the method being tested + app, err := client.GetApp(context.Background()) + + // Check the error + if tt.wantErr != nil { + assert.Error(t, err) + assert.Equal(t, tt.wantApp, app) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.wantApp, app) + } + }) + } +} + +func TestGithubClient_GetAppInstallation(t *testing.T) { + tests := []struct { + name string + mockHandler *http.Client + appToken string + installationID string + wantInstallation conngh.AppInstallation + wantErr bool + errContains string + }{ + { + name: "get disabled app installation successfully", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetAppInstallationsByInstallationId, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + installation := &github.Installation{ + ID: github.Ptr(int64(67890)), + SuspendedAt: github.Ptr(github.Timestamp{Time: time.Now()}), + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(installation)) + }), + ), + ), + appToken: "test-app-token", + installationID: "67890", + wantInstallation: conngh.AppInstallation{ + ID: 67890, + Enabled: false, + }, + wantErr: false, + }, + { + name: "get enabled app installation successfully", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetAppInstallationsByInstallationId, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + installation := &github.Installation{ + ID: github.Ptr(int64(67890)), + SuspendedAt: nil, + } + w.WriteHeader(http.StatusOK) + require.NoError(t, json.NewEncoder(w).Encode(installation)) + }), + ), + ), + appToken: "test-app-token", + installationID: "67890", + wantInstallation: conngh.AppInstallation{ + ID: 67890, + Enabled: true, + }, + wantErr: false, + }, + { + name: "invalid installation ID", + mockHandler: mockhub.NewMockedHTTPClient(), + appToken: "test-app-token", + installationID: "not-a-number", + wantInstallation: conngh.AppInstallation{}, + wantErr: true, + errContains: "invalid installation ID", + }, + { + name: "service unavailable", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetAppInstallationsByInstallationId, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusServiceUnavailable, + }, + Message: "Service unavailable", + })) + }), + ), + ), + appToken: "test-app-token", + installationID: "67890", + wantInstallation: conngh.AppInstallation{}, + wantErr: true, + }, + { + name: "installation not found", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetAppInstallationsByInstallationId, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusNotFound, + }, + Message: "Not Found", + })) + }), + ), + ), + appToken: "test-app-token", + installationID: "99999", + wantInstallation: conngh.AppInstallation{}, + wantErr: true, + }, + { + name: "other error", + mockHandler: mockhub.NewMockedHTTPClient( + mockhub.WithRequestMatchHandler( + mockhub.GetAppInstallationsByInstallationId, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusInternalServerError, + }, + Message: "Internal server error", + })) + }), + ), + ), + appToken: "test-app-token", + installationID: "67890", + wantInstallation: conngh.AppInstallation{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a mock client + ghClient := github.NewClient(tt.mockHandler) + client := conngh.NewClient(ghClient) + + // Call the method being tested + installation, err := client.GetAppInstallation(context.Background(), tt.installationID) + + // Check the error + if tt.wantErr { + assert.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + } else { + assert.NoError(t, err) + } + + // Check the result + assert.Equal(t, tt.wantInstallation, installation) + }) + } +} diff --git a/apps/provisioning/pkg/connection/github/connection.go b/apps/provisioning/pkg/connection/github/connection.go new file mode 100644 index 00000000000..6a2da98ac8d --- /dev/null +++ b/apps/provisioning/pkg/connection/github/connection.go @@ -0,0 +1,192 @@ +package github + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "time" + + "github.com/golang-jwt/jwt/v4" + provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/apps/provisioning/pkg/connection" + common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +//go:generate mockery --name GithubFactory --structname MockGithubFactory --inpackage --filename factory_mock.go --with-expecter +type GithubFactory interface { + New(ctx context.Context, ghToken common.RawSecureValue) Client +} + +type Connection struct { + obj *provisioning.Connection + ghFactory GithubFactory +} + +func NewConnection( + obj *provisioning.Connection, + factory GithubFactory, +) Connection { + return Connection{ + obj: obj, + ghFactory: factory, + } +} + +const ( + //TODO(ferruvich): these probably need to be setup in API configuration. + githubInstallationURL = "https://github.com/settings/installations" + jwtExpirationMinutes = 10 // GitHub Apps JWT tokens expire in 10 minutes maximum +) + +// Mutate performs in place mutation of the underneath resource. +func (c *Connection) Mutate(_ context.Context) error { + // Do nothing in case spec.Github is nil. + // If this field is required, we should fail at validation time. + if c.obj.Spec.GitHub == nil { + return nil + } + + c.obj.Spec.URL = fmt.Sprintf("%s/%s", githubInstallationURL, c.obj.Spec.GitHub.InstallationID) + + // Generate JWT token if private key is being provided. + // Same as for the spec.Github, if such a field is required, Validation will take care of that. + if !c.obj.Secure.PrivateKey.Create.IsZero() { + token, err := generateToken(c.obj.Spec.GitHub.AppID, c.obj.Secure.PrivateKey.Create) + if err != nil { + return fmt.Errorf("failed to generate JWT token: %w", err) + } + + // Store the generated token + c.obj.Secure.Token = common.InlineSecureValue{Create: token} + } + + return nil +} + +// Token generates and returns the Connection token. +func generateToken(appID string, privateKey common.RawSecureValue) (common.RawSecureValue, error) { + // Decode base64-encoded private key + privateKeyPEM, err := base64.StdEncoding.DecodeString(string(privateKey)) + if err != nil { + return "", fmt.Errorf("failed to decode base64 private key: %w", err) + } + + // Parse the private key + key, err := jwt.ParseRSAPrivateKeyFromPEM(privateKeyPEM) + if err != nil { + return "", fmt.Errorf("failed to parse private key: %w", err) + } + + // Create the JWT token + now := time.Now() + claims := jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(now.Add(time.Duration(jwtExpirationMinutes) * time.Minute)), + Issuer: appID, + } + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + signedToken, err := token.SignedString(key) + if err != nil { + return "", fmt.Errorf("failed to sign JWT token: %w", err) + } + + return common.RawSecureValue(signedToken), nil +} + +// Validate ensures the resource _looks_ correct. +func (c *Connection) Validate(ctx context.Context) error { + list := field.ErrorList{} + + if c.obj.Spec.Type != provisioning.GithubConnectionType { + list = append(list, field.Invalid(field.NewPath("spec", "type"), c.obj.Spec.Type, "invalid connection type")) + + // Doesn't make much sense to continue validating a connection which is not a Github one. + return toError(c.obj.GetName(), list) + } + + if c.obj.Spec.GitHub == nil { + list = append( + list, field.Required(field.NewPath("spec", "github"), "github info must be specified for GitHub connection"), + ) + + // Doesn't make much sense to continue validating a connection with no information. + return toError(c.obj.GetName(), list) + } + + if c.obj.Secure.PrivateKey.IsZero() { + list = append(list, field.Required(field.NewPath("secure", "privateKey"), "privateKey must be specified for GitHub connection")) + } + if c.obj.Secure.Token.IsZero() { + list = append(list, field.Required(field.NewPath("secure", "token"), "token must be specified for GitHub connection")) + } + if !c.obj.Secure.ClientSecret.IsZero() { + list = append(list, field.Forbidden(field.NewPath("secure", "clientSecret"), "clientSecret is forbidden in GitHub connection")) + } + + // Validate GitHub configuration fields + if c.obj.Spec.GitHub.AppID == "" { + list = append(list, field.Required(field.NewPath("spec", "github", "appID"), "appID must be specified for GitHub connection")) + } + if c.obj.Spec.GitHub.InstallationID == "" { + list = append(list, field.Required(field.NewPath("spec", "github", "installationID"), "installationID must be specified for GitHub connection")) + } + + // In case we have any error above, we don't go forward with the validation, and return the errors. + if len(list) > 0 { + return toError(c.obj.GetName(), list) + } + + // Validating app content via GH API + if err := c.validateAppAndInstallation(ctx); err != nil { + list = append(list, err) + } + + return toError(c.obj.GetName(), list) +} + +// validateAppAndInstallation validates the appID and installationID against the given github token. +func (c *Connection) validateAppAndInstallation(ctx context.Context) *field.Error { + ghClient := c.ghFactory.New(ctx, c.obj.Secure.Token.Create) + + app, err := ghClient.GetApp(ctx) + if err != nil { + if errors.Is(err, ErrServiceUnavailable) { + return field.InternalError(field.NewPath("spec", "token"), ErrServiceUnavailable) + } + return field.Invalid(field.NewPath("spec", "token"), "[REDACTED]", "invalid token") + } + + if fmt.Sprintf("%d", app.ID) != c.obj.Spec.GitHub.AppID { + return field.Invalid(field.NewPath("spec", "appID"), c.obj.Spec.GitHub.AppID, "appID mismatch") + } + + _, err = ghClient.GetAppInstallation(ctx, c.obj.Spec.GitHub.InstallationID) + if err != nil { + if errors.Is(err, ErrServiceUnavailable) { + return field.InternalError(field.NewPath("spec", "token"), ErrServiceUnavailable) + } + return field.Invalid(field.NewPath("spec", "installationID"), c.obj.Spec.GitHub.InstallationID, "invalid installation ID") + } + + return nil +} + +// toError converts a field.ErrorList to an error, returning nil if the list is empty +func toError(name string, list field.ErrorList) error { + if len(list) == 0 { + return nil + } + return apierrors.NewInvalid( + provisioning.ConnectionResourceInfo.GroupVersionKind().GroupKind(), + name, + list, + ) +} + +var ( + _ connection.Connection = (*Connection)(nil) +) diff --git a/apps/provisioning/pkg/connection/github/connection_test.go b/apps/provisioning/pkg/connection/github/connection_test.go new file mode 100644 index 00000000000..6a916db730e --- /dev/null +++ b/apps/provisioning/pkg/connection/github/connection_test.go @@ -0,0 +1,434 @@ +package github + +import ( + "context" + "encoding/base64" + "testing" + + provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//nolint:gosec // Test RSA private key (generated for testing purposes only) +const testPrivateKeyPEM = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAoInVbLY9io2Q/wHvUIXlEHg2Qyvd8eRzBAVEJ92DS6fx9H10 +06V0VRm78S0MXyo6i+n8ZAbZ0/R+GWpP2Ephxm0Gs2zo+iO2mpB19xQFI4o6ZTOw +b2WyjSaa2Vr4oyDkqti6AvfjW4VUAu932e08GkgwmmQSHXj7FX2CMWjgUwTTcuaX +65SHNKLNYLUP0HTumLzoZeqDTdoMMpKNdgH9Avr4/8vkVJ0mD6rqvxnw3JHsseNO +WdQTxf2aApBNHIIKxWZ2i/ZmjLNey7kltgjEquGiBdJvip3fHhH5XHdkrXcjRtnw +OJDnDmi5lQwv5yUBOSkbvbXRv/L/m0YLoD/fbwIDAQABAoIBAFfl//hM8/cnuesV ++R1Con/ZAgTXQOdPqPXbmEyniVrkMqMmCdBUOBTcST4s5yg36+RtkeaGpb/ajyyF +PAB2AYDucwvMpudGpJWOYTiOOp4R8hU1LvZfXVrRd1lo6NgQi4NLtNUpOtACeVQ+ +H4Yv0YemXQ47mnuOoRNMK/u3q5NoIdSahWptXBgUno8KklNpUrH3IYWaUxfBzDN3 +2xsVRTn2SfTSyoDmTDdTgptJONmoK1/sV7UsgWksdFc6XyYhsFAZgOGEJrBABRvF +546dyQ0cWxuPyVXpM7CN3tqC5ssvLjElg3LicK1V6gnjpdRnnvX88d1Eh3Uc/9IM +OZInT2ECgYEA6W8sQXTWinyEwl8SDKKMbB2ApIghAcFgdRxprZE4WFxjsYNCNL70 +dnSB7MRuzmxf5W77cV0N7JhH66N8HvY6Xq9olrpQ5dNttR4w8Pyv3wavDe8x7seL +5L2Xtbu7ihDr8Dk27MjiBSin3IxhBP5CJS910+pR6LrAWtEuU+FzFfECgYEAsA6y +qxHhCMXlTnauXhsnmPd1g61q7chW8kLQFYtHMLlQlgjHTW7irDZ9cPbPYDNjwRLO +7KLorcpv2NKe7rqq2ZyCm6hf1b9WnlQjo3dLpNWMu6fhy/smK8MgbRqcWpX+oTKF +79mK6hbY7o6eBzsQHBl7Z+LBNuwYmp9qOodPa18CgYEArv6ipKdcNhFGzRfMRiCN +OHederp6VACNuP2F05IsNUF9kxOdTEFirnKE++P+VU01TqA2azOhPp6iO+ohIGzi +MR06QNSH1OL9OWvasK4dggpWrRGF00VQgDgJRTnpS4WH+lxJ6pRlrAxgWpv6F24s +VAgSQr1Ejj2B+hMasdMvHWECgYBJ4uE4yhgXBnZlp4kmFV9Y4wF+cZkekaVrpn6N +jBYkbKFVVfnOlWqru3KJpgsB5I9IyAvvY68iwIKQDFSG+/AXw4dMrC0MF3DSoZ0T +TU2Br92QI7SvVod+djV1lGVp3ukt3XY4YqPZ+hywgUnw3uiz4j3YK2HLGup4ec6r +IX5DIQKBgHRLzvT3zqtlR1Oh0vv098clLwt+pGzXOxzJpxioOa5UqK13xIpFXbcg +iWUVh5YXCcuqaICUv4RLIEac5xQitk9Is/9IhP0NJ/81rHniosvdSpCeFXzxTImS +B8Uc0WUgheB4+yVKGnYpYaSOgFFI5+1BYUva/wDHLy2pWHz39Usb +-----END RSA PRIVATE KEY-----` + +func TestConnection_Mutate(t *testing.T) { + t.Run("should add URL to Github connection", func(t *testing.T) { + c := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Name: "test-private-key", + }, + }, + } + + mockFactory := NewMockGithubFactory(t) + conn := NewConnection(c, mockFactory) + + require.NoError(t, conn.Mutate(context.Background())) + assert.Equal(t, "https://github.com/settings/installations/456", c.Spec.URL) + }) + + t.Run("should generate JWT token when private key is provided", func(t *testing.T) { + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) + + c := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue(privateKeyBase64), + }, + }, + } + + mockFactory := NewMockGithubFactory(t) + conn := NewConnection(c, mockFactory) + + require.NoError(t, conn.Mutate(context.Background())) + assert.Equal(t, "https://github.com/settings/installations/456", c.Spec.URL) + assert.False(t, c.Secure.Token.Create.IsZero(), "JWT token should be generated") + }) + + t.Run("should do nothing when GitHub config is nil", func(t *testing.T) { + c := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GitlabConnectionType, + Gitlab: &provisioning.GitlabConnectionConfig{ + ClientID: "clientID", + }, + }, + } + + mockFactory := NewMockGithubFactory(t) + conn := NewConnection(c, mockFactory) + + require.NoError(t, conn.Mutate(context.Background())) + }) + + t.Run("should fail when private key is not base64", func(t *testing.T) { + c := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("invalid-key"), + }, + }, + } + + mockFactory := NewMockGithubFactory(t) + conn := NewConnection(c, mockFactory) + + err := conn.Mutate(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to generate JWT token") + assert.Contains(t, err.Error(), "failed to decode base64 private key") + }) + + t.Run("should fail when private key is invalid", func(t *testing.T) { + c := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue(base64.StdEncoding.EncodeToString([]byte("invalid-key"))), + }, + }, + } + + mockFactory := NewMockGithubFactory(t) + conn := NewConnection(c, mockFactory) + + err := conn.Mutate(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to generate JWT token") + assert.Contains(t, err.Error(), "failed to parse private key") + }) +} + +func TestConnection_Validate(t *testing.T) { + tests := []struct { + name string + connection *provisioning.Connection + setupMock func(*MockGithubFactory) + wantErr bool + errMsgContains []string + }{ + { + name: "invalid type returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: "invalid", + }, + }, + wantErr: true, + errMsgContains: []string{"spec.type"}, + }, + { + name: "github type without github config returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + }, + }, + wantErr: true, + errMsgContains: []string{"spec.github"}, + }, + { + name: "github type without private key returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + }, + wantErr: true, + errMsgContains: []string{"secure.privateKey"}, + }, + { + name: "github type without token returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + }, + }, + wantErr: true, + errMsgContains: []string{"secure.token"}, + }, + { + name: "github type with client secret returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + ClientSecret: common.InlineSecureValue{ + Create: common.NewSecretValue("test-client-secret"), + }, + }, + }, + wantErr: true, + errMsgContains: []string{"secure.clientSecret"}, + }, + { + name: "github type without appID returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + Token: common.InlineSecureValue{ + Create: common.NewSecretValue("test-token"), + }, + }, + }, + wantErr: true, + errMsgContains: []string{"spec.github.appID"}, + }, + { + name: "github type without installationID returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Name: "test-private-key", + }, + Token: common.InlineSecureValue{ + Name: "test-token", + }, + }, + }, + wantErr: true, + errMsgContains: []string{"spec.github.installationID"}, + }, + { + name: "github type with valid config is valid", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + Token: common.InlineSecureValue{ + Create: common.NewSecretValue("test-token"), + }, + }, + }, + wantErr: false, + setupMock: func(mockFactory *MockGithubFactory) { + mockClient := NewMockClient(t) + + mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient) + mockClient.EXPECT().GetApp(mock.Anything).Return(App{ID: 123, Slug: "test-app"}, nil) + mockClient.EXPECT().GetAppInstallation(mock.Anything, "456").Return(AppInstallation{ID: 456}, nil) + }, + }, + { + name: "problem getting app returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + Token: common.InlineSecureValue{ + Create: common.NewSecretValue("test-token"), + }, + }, + }, + wantErr: true, + errMsgContains: []string{"spec.token", "[REDACTED]"}, + setupMock: func(mockFactory *MockGithubFactory) { + mockClient := NewMockClient(t) + + mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient) + mockClient.EXPECT().GetApp(mock.Anything).Return(App{}, assert.AnError) + }, + }, + { + name: "mismatched app ID returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + Token: common.InlineSecureValue{ + Create: common.NewSecretValue("test-token"), + }, + }, + }, + wantErr: true, + errMsgContains: []string{"spec.appID"}, + setupMock: func(mockFactory *MockGithubFactory) { + mockClient := NewMockClient(t) + + mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient) + mockClient.EXPECT().GetApp(mock.Anything).Return(App{ID: 444, Slug: "test-app"}, nil) + }, + }, + { + name: "problem when getting installation returns error", + connection: &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + Token: common.InlineSecureValue{ + Create: common.NewSecretValue("test-token"), + }, + }, + }, + wantErr: true, + errMsgContains: []string{"spec.installationID", "456"}, + setupMock: func(mockFactory *MockGithubFactory) { + mockClient := NewMockClient(t) + + mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient) + mockClient.EXPECT().GetApp(mock.Anything).Return(App{ID: 123, Slug: "test-app"}, nil) + mockClient.EXPECT().GetAppInstallation(mock.Anything, "456").Return(AppInstallation{}, assert.AnError) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockFactory := NewMockGithubFactory(t) + if tt.setupMock != nil { + tt.setupMock(mockFactory) + } + + conn := NewConnection(tt.connection, mockFactory) + err := conn.Validate(context.Background()) + if tt.wantErr { + assert.Error(t, err) + for _, msg := range tt.errMsgContains { + assert.Contains(t, err.Error(), msg) + } + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/apps/provisioning/pkg/connection/github/extra.go b/apps/provisioning/pkg/connection/github/extra.go new file mode 100644 index 00000000000..2c207637c61 --- /dev/null +++ b/apps/provisioning/pkg/connection/github/extra.go @@ -0,0 +1,36 @@ +package github + +import ( + "context" + "fmt" + + "github.com/grafana/grafana-app-sdk/logging" + provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/apps/provisioning/pkg/connection" +) + +type extra struct { + factory GithubFactory +} + +func (e *extra) Type() provisioning.ConnectionType { + return provisioning.GithubConnectionType +} + +func (e *extra) Build(ctx context.Context, connection *provisioning.Connection) (connection.Connection, error) { + logger := logging.FromContext(ctx) + if connection == nil || connection.Spec.GitHub == nil { + logger.Error("connection is nil or github info is nil") + + return nil, fmt.Errorf("invalid github connection") + } + + c := NewConnection(connection, e.factory) + return &c, nil +} + +func Extra(factory GithubFactory) connection.Extra { + return &extra{ + factory: factory, + } +} diff --git a/apps/provisioning/pkg/connection/github/extra_test.go b/apps/provisioning/pkg/connection/github/extra_test.go new file mode 100644 index 00000000000..c5bcc8279d9 --- /dev/null +++ b/apps/provisioning/pkg/connection/github/extra_test.go @@ -0,0 +1,126 @@ +package github_test + +import ( + "context" + "testing" + + provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" + common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestExtra_Type(t *testing.T) { + t.Run("should return GithubConnectionType", func(t *testing.T) { + mockFactory := github.NewMockGithubFactory(t) + e := github.Extra(mockFactory) + result := e.Type() + assert.Equal(t, provisioning.GithubConnectionType, result) + }) +} + +func TestExtra_Build(t *testing.T) { + t.Run("should successfully build connection", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Create: common.NewSecretValue("test-private-key"), + }, + }, + } + + mockFactory := github.NewMockGithubFactory(t) + + e := github.Extra(mockFactory) + + result, err := e.Build(ctx, conn) + require.NoError(t, err) + require.NotNil(t, result) + }) + + t.Run("should handle different connection configurations", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "another-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "789", + InstallationID: "101112", + }, + }, + Secure: provisioning.ConnectionSecure{ + PrivateKey: common.InlineSecureValue{ + Name: "existing-private-key", + }, + Token: common.InlineSecureValue{ + Name: "existing-token", + }, + }, + } + + mockFactory := github.NewMockGithubFactory(t) + + e := github.Extra(mockFactory) + + result, err := e.Build(ctx, conn) + require.NoError(t, err) + require.NotNil(t, result) + }) + + t.Run("should build connection with background context", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + } + + mockFactory := github.NewMockGithubFactory(t) + e := github.Extra(mockFactory) + result, err := e.Build(ctx, conn) + require.NoError(t, err) + require.NotNil(t, result) + }) + + t.Run("should always pass empty token to factory.New", func(t *testing.T) { + ctx := context.Background() + conn := &provisioning.Connection{ + ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, + Spec: provisioning.ConnectionSpec{ + Type: provisioning.GithubConnectionType, + GitHub: &provisioning.GitHubConnectionConfig{ + AppID: "123", + InstallationID: "456", + }, + }, + Secure: provisioning.ConnectionSecure{ + Token: common.InlineSecureValue{ + Create: common.NewSecretValue("some-token"), + }, + }, + } + + mockFactory := github.NewMockGithubFactory(t) + e := github.Extra(mockFactory) + result, err := e.Build(ctx, conn) + require.NoError(t, err) + require.NotNil(t, result) + }) +} diff --git a/apps/provisioning/pkg/connection/github/factory.go b/apps/provisioning/pkg/connection/github/factory.go new file mode 100644 index 00000000000..2399f3c9f69 --- /dev/null +++ b/apps/provisioning/pkg/connection/github/factory.go @@ -0,0 +1,39 @@ +package github + +import ( + "context" + "net/http" + + "github.com/google/go-github/v70/github" + "golang.org/x/oauth2" + + common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" +) + +// Factory creates new GitHub clients. +// It exists only for the ability to test the code easily. +type Factory struct { + // Client allows overriding the client to use in the GH client returned. It exists primarily for testing. + // FIXME: we should replace in this way. We should add some options pattern for the factory. + Client *http.Client +} + +func ProvideFactory() GithubFactory { + return &Factory{} +} + +func (r *Factory) New(ctx context.Context, ghToken common.RawSecureValue) Client { + if r.Client != nil { + return NewClient(github.NewClient(r.Client)) + } + + if !ghToken.IsZero() { + tokenSrc := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: string(ghToken)}, + ) + tokenClient := oauth2.NewClient(ctx, tokenSrc) + return NewClient(github.NewClient(tokenClient)) + } + + return NewClient(github.NewClient(&http.Client{})) +} diff --git a/apps/provisioning/pkg/connection/github/factory_mock.go b/apps/provisioning/pkg/connection/github/factory_mock.go new file mode 100644 index 00000000000..a9e1424b62d --- /dev/null +++ b/apps/provisioning/pkg/connection/github/factory_mock.go @@ -0,0 +1,86 @@ +// Code generated by mockery v2.53.4. DO NOT EDIT. + +package github + +import ( + context "context" + + v0alpha1 "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" + mock "github.com/stretchr/testify/mock" +) + +// MockGithubFactory is an autogenerated mock type for the GithubFactory type +type MockGithubFactory struct { + mock.Mock +} + +type MockGithubFactory_Expecter struct { + mock *mock.Mock +} + +func (_m *MockGithubFactory) EXPECT() *MockGithubFactory_Expecter { + return &MockGithubFactory_Expecter{mock: &_m.Mock} +} + +// New provides a mock function with given fields: ctx, ghToken +func (_m *MockGithubFactory) New(ctx context.Context, ghToken v0alpha1.RawSecureValue) Client { + ret := _m.Called(ctx, ghToken) + + if len(ret) == 0 { + panic("no return value specified for New") + } + + var r0 Client + if rf, ok := ret.Get(0).(func(context.Context, v0alpha1.RawSecureValue) Client); ok { + r0 = rf(ctx, ghToken) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(Client) + } + } + + return r0 +} + +// MockGithubFactory_New_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'New' +type MockGithubFactory_New_Call struct { + *mock.Call +} + +// New is a helper method to define mock.On call +// - ctx context.Context +// - ghToken v0alpha1.RawSecureValue +func (_e *MockGithubFactory_Expecter) New(ctx interface{}, ghToken interface{}) *MockGithubFactory_New_Call { + return &MockGithubFactory_New_Call{Call: _e.mock.On("New", ctx, ghToken)} +} + +func (_c *MockGithubFactory_New_Call) Run(run func(ctx context.Context, ghToken v0alpha1.RawSecureValue)) *MockGithubFactory_New_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(v0alpha1.RawSecureValue)) + }) + return _c +} + +func (_c *MockGithubFactory_New_Call) Return(_a0 Client) *MockGithubFactory_New_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockGithubFactory_New_Call) RunAndReturn(run func(context.Context, v0alpha1.RawSecureValue) Client) *MockGithubFactory_New_Call { + _c.Call.Return(run) + return _c +} + +// NewMockGithubFactory creates a new instance of MockGithubFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockGithubFactory(t interface { + mock.TestingT + Cleanup(func()) +}) *MockGithubFactory { + mock := &MockGithubFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/apps/provisioning/pkg/connection/mutator.go b/apps/provisioning/pkg/connection/mutator.go deleted file mode 100644 index 30291669905..00000000000 --- a/apps/provisioning/pkg/connection/mutator.go +++ /dev/null @@ -1,28 +0,0 @@ -package connection - -import ( - "fmt" - - provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" -) - -const ( - githubInstallationURL = "https://github.com/settings/installations" -) - -func MutateConnection(connection *provisioning.Connection) error { - switch connection.Spec.Type { - case provisioning.GithubConnectionType: - // Do nothing in case spec.Github is nil. - // If this field is required, we should fail at validation time. - if connection.Spec.GitHub == nil { - return nil - } - - connection.Spec.URL = fmt.Sprintf("%s/%s", githubInstallationURL, connection.Spec.GitHub.InstallationID) - return nil - default: - // TODO: we need to setup the URL for bitbucket and gitlab. - return nil - } -} diff --git a/apps/provisioning/pkg/connection/mutator_test.go b/apps/provisioning/pkg/connection/mutator_test.go deleted file mode 100644 index a25aabd10a1..00000000000 --- a/apps/provisioning/pkg/connection/mutator_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package connection_test - -import ( - "testing" - - provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" - "github.com/grafana/grafana/apps/provisioning/pkg/connection" - common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestMutateConnection(t *testing.T) { - t.Run("should add URL to Github connection", func(t *testing.T) { - c := &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GithubConnectionType, - GitHub: &provisioning.GitHubConnectionConfig{ - AppID: "123", - InstallationID: "456", - }, - }, - Secure: provisioning.ConnectionSecure{ - PrivateKey: common.InlineSecureValue{ - Name: "test-private-key", - }, - }, - } - - require.NoError(t, connection.MutateConnection(c)) - assert.Equal(t, "https://github.com/settings/installations/456", c.Spec.URL) - }) -} diff --git a/apps/provisioning/pkg/connection/validator.go b/apps/provisioning/pkg/connection/validator.go deleted file mode 100644 index c2537e3af2f..00000000000 --- a/apps/provisioning/pkg/connection/validator.go +++ /dev/null @@ -1,104 +0,0 @@ -package connection - -import ( - provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -func ValidateConnection(connection *provisioning.Connection) error { - list := field.ErrorList{} - - if connection.Spec.Type == "" { - list = append(list, field.Required(field.NewPath("spec", "type"), "type must be specified")) - } - - switch connection.Spec.Type { - case provisioning.GithubConnectionType: - list = append(list, validateGithubConnection(connection)...) - case provisioning.BitbucketConnectionType: - list = append(list, validateBitbucketConnection(connection)...) - case provisioning.GitlabConnectionType: - list = append(list, validateGitlabConnection(connection)...) - default: - list = append( - list, field.NotSupported( - field.NewPath("spec", "type"), - connection.Spec.Type, - []provisioning.ConnectionType{ - provisioning.GithubConnectionType, - provisioning.BitbucketConnectionType, - provisioning.GitlabConnectionType, - }), - ) - } - - return toError(connection.GetName(), list) -} - -func validateGithubConnection(connection *provisioning.Connection) field.ErrorList { - list := field.ErrorList{} - - if connection.Spec.GitHub == nil { - list = append( - list, field.Required(field.NewPath("spec", "github"), "github info must be specified for GitHub connection"), - ) - } - - if connection.Secure.PrivateKey.IsZero() { - list = append(list, field.Required(field.NewPath("secure", "privateKey"), "privateKey must be specified for GitHub connection")) - } - if !connection.Secure.ClientSecret.IsZero() { - list = append(list, field.Forbidden(field.NewPath("secure", "clientSecret"), "clientSecret is forbidden in GitHub connection")) - } - - return list -} - -func validateBitbucketConnection(connection *provisioning.Connection) field.ErrorList { - list := field.ErrorList{} - - if connection.Spec.Bitbucket == nil { - list = append( - list, field.Required(field.NewPath("spec", "bitbucket"), "bitbucket info must be specified in Bitbucket connection"), - ) - } - if connection.Secure.ClientSecret.IsZero() { - list = append(list, field.Required(field.NewPath("secure", "clientSecret"), "clientSecret must be specified for Bitbucket connection")) - } - if !connection.Secure.PrivateKey.IsZero() { - list = append(list, field.Forbidden(field.NewPath("secure", "privateKey"), "privateKey is forbidden in Bitbucket connection")) - } - - return list -} - -func validateGitlabConnection(connection *provisioning.Connection) field.ErrorList { - list := field.ErrorList{} - - if connection.Spec.Gitlab == nil { - list = append( - list, field.Required(field.NewPath("spec", "gitlab"), "gitlab info must be specified in Gitlab connection"), - ) - } - if connection.Secure.ClientSecret.IsZero() { - list = append(list, field.Required(field.NewPath("secure", "clientSecret"), "clientSecret must be specified for Gitlab connection")) - } - if !connection.Secure.PrivateKey.IsZero() { - list = append(list, field.Forbidden(field.NewPath("secure", "privateKey"), "privateKey is forbidden in Gitlab connection")) - } - - return list -} - -// toError converts a field.ErrorList to an error, returning nil if the list is empty -func toError(name string, list field.ErrorList) error { - if len(list) == 0 { - return nil - } - return apierrors.NewInvalid( - provisioning.ConnectionResourceInfo.GroupVersionKind().GroupKind(), - name, - list, - ) -} diff --git a/apps/provisioning/pkg/connection/validator_test.go b/apps/provisioning/pkg/connection/validator_test.go deleted file mode 100644 index 23d4b01b800..00000000000 --- a/apps/provisioning/pkg/connection/validator_test.go +++ /dev/null @@ -1,253 +0,0 @@ -package connection_test - -import ( - "testing" - - provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" - "github.com/grafana/grafana/apps/provisioning/pkg/connection" - common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" - "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestValidateConnection(t *testing.T) { - tests := []struct { - name string - connection *provisioning.Connection - wantErr bool - errMsg string - }{ - { - name: "empty type returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{}, - }, - wantErr: true, - errMsg: "spec.type", - }, - { - name: "invalid type returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: "invalid", - }, - }, - wantErr: true, - errMsg: "spec.type", - }, - { - name: "github type without github config returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GithubConnectionType, - }, - }, - wantErr: true, - errMsg: "spec.github", - }, - { - name: "github type without private key returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GithubConnectionType, - GitHub: &provisioning.GitHubConnectionConfig{ - AppID: "123", - InstallationID: "456", - }, - }, - }, - wantErr: true, - errMsg: "secure.privateKey", - }, - { - name: "github type with client secret returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GithubConnectionType, - GitHub: &provisioning.GitHubConnectionConfig{ - AppID: "123", - InstallationID: "456", - }, - }, - Secure: provisioning.ConnectionSecure{ - PrivateKey: common.InlineSecureValue{ - Name: "test-private-key", - }, - ClientSecret: common.InlineSecureValue{ - Name: "test-client-secret", - }, - }, - }, - wantErr: true, - errMsg: "secure.clientSecret", - }, - { - name: "github type with github config is valid", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GithubConnectionType, - GitHub: &provisioning.GitHubConnectionConfig{ - AppID: "123", - InstallationID: "456", - }, - }, - Secure: provisioning.ConnectionSecure{ - PrivateKey: common.InlineSecureValue{ - Name: "test-private-key", - }, - }, - }, - wantErr: false, - }, - { - name: "bitbucket type without bitbucket config returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.BitbucketConnectionType, - }, - }, - wantErr: true, - errMsg: "spec.bitbucket", - }, - { - name: "bitbucket type without client secret returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.BitbucketConnectionType, - Bitbucket: &provisioning.BitbucketConnectionConfig{ - ClientID: "client-123", - }, - }, - }, - wantErr: true, - errMsg: "secure.clientSecret", - }, - { - name: "bitbucket type with private key returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.BitbucketConnectionType, - Bitbucket: &provisioning.BitbucketConnectionConfig{ - ClientID: "client-123", - }, - }, - Secure: provisioning.ConnectionSecure{ - PrivateKey: common.InlineSecureValue{ - Name: "test-private-key", - }, - ClientSecret: common.InlineSecureValue{ - Name: "test-client-secret", - }, - }, - }, - wantErr: true, - errMsg: "secure.privateKey", - }, - { - name: "bitbucket type with bitbucket config is valid", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.BitbucketConnectionType, - Bitbucket: &provisioning.BitbucketConnectionConfig{ - ClientID: "client-123", - }, - }, - Secure: provisioning.ConnectionSecure{ - ClientSecret: common.InlineSecureValue{ - Name: "test-client-secret", - }, - }, - }, - wantErr: false, - }, - { - name: "gitlab type without gitlab config returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GitlabConnectionType, - }, - }, - wantErr: true, - errMsg: "spec.gitlab", - }, - { - name: "gitlab type without client secret returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GitlabConnectionType, - Gitlab: &provisioning.GitlabConnectionConfig{ - ClientID: "client-456", - }, - }, - }, - wantErr: true, - errMsg: "secure.clientSecret", - }, - { - name: "gitlab type with private key returns error", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GitlabConnectionType, - Gitlab: &provisioning.GitlabConnectionConfig{ - ClientID: "client-456", - }, - }, - Secure: provisioning.ConnectionSecure{ - PrivateKey: common.InlineSecureValue{ - Name: "test-private-key", - }, - ClientSecret: common.InlineSecureValue{ - Name: "test-client-secret", - }, - }, - }, - wantErr: true, - errMsg: "secure.privateKey", - }, - { - name: "gitlab type with gitlab config is valid", - connection: &provisioning.Connection{ - ObjectMeta: metav1.ObjectMeta{Name: "test-connection"}, - Spec: provisioning.ConnectionSpec{ - Type: provisioning.GitlabConnectionType, - Gitlab: &provisioning.GitlabConnectionConfig{ - ClientID: "client-456", - }, - }, - Secure: provisioning.ConnectionSecure{ - ClientSecret: common.InlineSecureValue{ - Name: "test-client-secret", - }, - }, - }, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := connection.ValidateConnection(tt.connection) - if tt.wantErr { - assert.Error(t, err) - if tt.errMsg != "" { - assert.Contains(t, err.Error(), tt.errMsg) - } - } else { - assert.NoError(t, err) - } - }) - } -} diff --git a/apps/provisioning/pkg/generated/applyconfiguration/provisioning/v0alpha1/connectionsecure.go b/apps/provisioning/pkg/generated/applyconfiguration/provisioning/v0alpha1/connectionsecure.go index 8ac26b192c9..f5be635560d 100644 --- a/apps/provisioning/pkg/generated/applyconfiguration/provisioning/v0alpha1/connectionsecure.go +++ b/apps/provisioning/pkg/generated/applyconfiguration/provisioning/v0alpha1/connectionsecure.go @@ -13,7 +13,7 @@ import ( type ConnectionSecureApplyConfiguration struct { PrivateKey *commonv0alpha1.InlineSecureValue `json:"privateKey,omitempty"` ClientSecret *commonv0alpha1.InlineSecureValue `json:"clientSecret,omitempty"` - Token *commonv0alpha1.InlineSecureValue `json:"webhook,omitempty"` + Token *commonv0alpha1.InlineSecureValue `json:"token,omitempty"` } // ConnectionSecureApplyConfiguration constructs a declarative configuration of the ConnectionSecure type for use with diff --git a/docs/sources/datasources/mssql/_index.md b/docs/sources/datasources/mssql/_index.md index a5e00da6dcb..b7ea159e972 100644 --- a/docs/sources/datasources/mssql/_index.md +++ b/docs/sources/datasources/mssql/_index.md @@ -99,12 +99,27 @@ refs: destination: /docs/grafana//administration/data-source-management/#query-and-resource-caching - pattern: /docs/grafana-cloud/ destination: /docs/grafana//administration/data-source-management/#query-and-resource-caching + mssql-troubleshoot: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mssql/troubleshooting/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mssql/troubleshooting/ + postgres: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/postgres/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/postgres/ + mysql: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mysql/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mysql/ --- # Microsoft SQL Server (MSSQL) data source Grafana ships with built-in support for Microsoft SQL Server (MSSQL). -You can query and visualize data from any Microsoft SQL Server 2005 or newer, including the Microsoft Azure SQL Database. +You can query and visualize data from any Microsoft SQL Server 2005 or newer, including Microsoft Azure SQL Database. Use this data source to create dashboards, explore SQL data, and monitor MSSQL-based workloads in real time. @@ -113,10 +128,33 @@ The following documentation helps you get started working with the Microsoft SQL - [Configure the Microsoft SQL Server data source](ref:configure-mssql-data-source) - [Microsoft SQL Server query editor](ref:mssql-query-editor) - [Microsoft SQL Server template variables](ref:mssql-template-variables) +- [Troubleshoot Microsoft SQL Server data source issues](ref:mssql-troubleshoot) -## Get the most out of the data source +## Supported versions -After installing and configuring the Microsoft SQL Server data source, you can: +This data source supports the following Microsoft SQL Server versions: + +- Microsoft SQL Server 2005 and newer +- Microsoft Azure SQL Database +- Azure SQL Managed Instance + +Grafana recommends using the latest available service pack for your SQL Server version for optimal compatibility. + +## Key capabilities + +The Microsoft SQL Server data source supports: + +- **Time series queries:** Visualize metrics over time using the built-in time grouping macros. +- **Table queries:** Display query results in table format for any valid SQL query. +- **Template variables:** Create dynamic dashboards with variable-driven queries. +- **Annotations:** Overlay events from SQL Server on your dashboard graphs. +- **Alerting:** Create alerts based on SQL Server query results. +- **Stored procedures:** Execute stored procedures and visualize results. +- **Macros:** Simplify queries with built-in macros for time filtering and grouping. + +## Additional resources + +After configuring the Microsoft SQL Server data source, you can: - Create a wide variety of [visualizations](ref:visualizations) - Configure and use [templates and variables](ref:variables) @@ -124,3 +162,8 @@ After installing and configuring the Microsoft SQL Server data source, you can: - Add [annotations](ref:annotate-visualizations) - Set up [alerting](ref:alerting) - Optimize performance with [query caching](ref:query-caching) + +## Related data sources + +- [PostgreSQL](ref:postgres) - For PostgreSQL databases. +- [MySQL](ref:mysql) - For MySQL and MariaDB databases. diff --git a/docs/sources/datasources/mssql/configure/index.md b/docs/sources/datasources/mssql/configure/index.md index 7ce6398f1cc..f41deeb51dd 100644 --- a/docs/sources/datasources/mssql/configure/index.md +++ b/docs/sources/datasources/mssql/configure/index.md @@ -89,6 +89,26 @@ refs: destination: /docs/grafana//setup-grafana/configure-access/configure-authentication/azuread/#enable-azure-ad-oauth-in-grafana - pattern: /docs/grafana-cloud/ destination: /docs/grafana//setup-grafana/configure-access/configure-authentication/azuread/#enable-azure-ad-oauth-in-grafana + mssql-query-editor: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mssql/query-editor/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mssql/query-editor/ + mssql-template-variables: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mssql/template-variables/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mssql/template-variables/ + alerting: + - pattern: /docs/grafana/ + destination: /docs/grafana//alerting/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana-cloud/alerting-and-irm/alerting/ + mssql-troubleshoot: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mssql/troubleshooting/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mssql/troubleshooting/ --- # Configure the Microsoft SQL Server data source @@ -97,13 +117,28 @@ This document provides instructions for configuring the Microsoft SQL Server dat ## Before you begin -- Grafana comes with a built-in MSSQL data source plugin, eliminating the need to install a plugin. +Before configuring the Microsoft SQL Server data source, ensure you have the following: -- You must have the `Organization administrator` role to configure the MSSQL data source. Organization administrators can also [configure the data source via YAML](#provision-the-data-source) with the Grafana provisioning system. +- **Grafana permissions:** You must have the `Organization administrator` role to configure data sources. Organization administrators can also [configure the data source via YAML](#provision-the-data-source) with the Grafana provisioning system. -- Familiarize yourself with your MSSQL security configuration and gather any necessary security certificates and client keys. +- **A running SQL Server instance:** Microsoft SQL Server 2005 or newer, Azure SQL Database, or Azure SQL Managed Instance. -- Verify that data from MSSQL is being written to your Grafana instance. +- **Network access:** Grafana must be able to reach your SQL Server. The default port is `1433`. + +- **Authentication credentials:** Depending on your authentication method, you need one of: + - SQL Server login credentials (username and password). + - Windows/Kerberos credentials and configuration (not supported in Grafana Cloud). + - Azure Entra ID app registration or managed identity. + +- **Security certificates:** If using encrypted connections, gather any necessary TLS/SSL certificates. + +{{< admonition type="note" >}} +Grafana ships with a built-in Microsoft SQL Server data source plugin. No additional installation is required. +{{< /admonition >}} + +{{< admonition type="tip" >}} +**Grafana Cloud users:** If your SQL Server is in a private network, you can configure [Private data source connect](ref:private-data-source-connect) to establish connectivity. +{{< /admonition >}} ## Add the MSSQL data source @@ -382,3 +417,48 @@ datasources: secureJsonData: password: 'Password!' ``` + +### Configure with Terraform + +You can configure the Microsoft SQL Server data source using [Terraform](https://www.terraform.io/) with the [Grafana Terraform provider](https://registry.terraform.io/providers/grafana/grafana/latest/docs). + +For more information about provisioning resources with Terraform, refer to the [Grafana as code using Terraform](https://grafana.com/docs/grafana-cloud/developer-resources/infrastructure-as-code/terraform/) documentation. + +#### Terraform example + +The following example creates a basic Microsoft SQL Server data source: + +```hcl +resource "grafana_data_source" "mssql" { + name = "MSSQL" + type = "mssql" + url = "localhost:1433" + user = "grafana" + + json_data_encoded = jsonencode({ + database = "grafana" + maxOpenConns = 100 + maxIdleConns = 100 + maxIdleConnsAuto = true + connMaxLifetime = 14400 + connectionTimeout = 0 + encrypt = "false" + }) + + secure_json_data_encoded = jsonencode({ + password = "Password!" + }) +} +``` + +For all available configuration options, refer to the [Grafana provider data source resource documentation](https://registry.terraform.io/providers/grafana/grafana/latest/docs/resources/data_source). + +## Next steps + +After configuring your Microsoft SQL Server data source, you can: + +- [Write queries](ref:mssql-query-editor) using the query editor to explore and visualize your data +- [Create template variables](ref:mssql-template-variables) to build dynamic, reusable dashboards +- [Add annotations](ref:annotate-visualizations) to overlay SQL Server events on your graphs +- [Set up alerting](ref:alerting) to create alert rules based on your SQL Server data +- [Troubleshoot issues](ref:mssql-troubleshoot) if you encounter problems with your data source diff --git a/docs/sources/datasources/mssql/troubleshooting/index.md b/docs/sources/datasources/mssql/troubleshooting/index.md new file mode 100644 index 00000000000..a62f3eb59e1 --- /dev/null +++ b/docs/sources/datasources/mssql/troubleshooting/index.md @@ -0,0 +1,333 @@ +--- +description: Troubleshoot common problems with the Microsoft SQL Server data source in Grafana +keywords: + - grafana + - MSSQL + - Microsoft + - SQL + - troubleshooting + - errors +labels: + products: + - cloud + - enterprise + - oss +menuTitle: Troubleshooting +title: Troubleshoot Microsoft SQL Server data source issues +weight: 400 +refs: + configure-mssql-data-source: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mssql/configure/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mssql/configure/ + mssql-query-editor: + - pattern: /docs/grafana/ + destination: /docs/grafana//datasources/mssql/query-editor/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana//datasources/mssql/query-editor/ + private-data-source-connect: + - pattern: /docs/grafana/ + destination: /docs/grafana-cloud/connect-externally-hosted/private-data-source-connect/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana-cloud/connect-externally-hosted/private-data-source-connect/ +--- + +# Troubleshoot Microsoft SQL Server data source issues + +This document provides solutions to common issues you may encounter when configuring or using the Microsoft SQL Server (MSSQL) data source in Grafana. + +## Connection errors + +These errors occur when Grafana cannot establish or maintain a connection to the Microsoft SQL Server. + +### Unable to connect to the server + +**Error message:** "Unable to open tcp connection" or "dial tcp: connection refused" + +**Cause:** Grafana cannot establish a network connection to the SQL Server. + +**Solution:** + +1. Verify that the SQL Server is running and accessible. +1. Check that the host and port are correct in the data source configuration. The default SQL Server port is `1433`. +1. Ensure there are no firewall rules blocking the connection between Grafana and SQL Server. +1. Verify that SQL Server is configured to allow remote connections. +1. For Grafana Cloud, ensure you have configured [Private data source connect](ref:private-data-source-connect) if your SQL Server instance is not publicly accessible. + +### Connection timeout + +**Error message:** "Connection timed out" or "I/O timeout" + +**Cause:** The connection to SQL Server timed out before receiving a response. + +**Solution:** + +1. Check the network latency between Grafana and SQL Server. +1. Verify that SQL Server is not overloaded or experiencing performance issues. +1. Increase the **Connection timeout** setting in the data source configuration under **Additional settings**. +1. Check if any network devices (load balancers, proxies) are timing out the connection. + +### Encryption-related connection failures + +**Error message:** "TLS handshake failed" or "certificate verify failed" + +**Cause:** There is a mismatch between the encryption settings in Grafana and what the SQL Server supports or requires. + +**Solution:** + +1. For older versions of SQL Server (2008, 2008R2), set the **Encrypt** option to **Disable** or **False** in the data source configuration. +1. Verify that the SQL Server has a valid SSL certificate if encryption is enabled. +1. Check that the certificate is trusted by the Grafana server. +1. Ensure you're using the latest available service pack for your SQL Server version for optimal compatibility. + +### Named instance connection issues + +**Error message:** "Cannot connect to named instance" or connection fails when using instance name + +**Cause:** Grafana cannot resolve the SQL Server named instance. + +**Solution:** + +1. Use the format `hostname\instancename` or `hostname\instancename,port` in the **Host** field. +1. Verify that the SQL Server Browser service is running on the SQL Server machine. +1. If the Browser service is unavailable, specify the port number directly: `hostname,port`. +1. Check that UDP port 1434 is open if using the SQL Server Browser service. + +## Authentication errors + +These errors occur when there are issues with authentication credentials or permissions. + +### Login failed for user + +**Error message:** "Login failed for user 'username'" or "Authentication failed" + +**Cause:** The authentication credentials are invalid or the user doesn't have permission to access the database. + +**Solution:** + +1. Verify that the username and password are correct. +1. Check that the user exists in SQL Server and is enabled. +1. Ensure the user has access to the specified database. +1. For Windows Authentication, verify that the credentials are in the correct format (`DOMAIN\User`). +1. Check that the SQL Server authentication mode allows the type of login you're using (SQL Server Authentication, Windows Authentication, or Mixed Mode). + +### Access denied to database + +**Error message:** "Cannot open database 'dbname' requested by the login" + +**Cause:** The authenticated user doesn't have permission to access the specified database. + +**Solution:** + +1. Verify that the database name is correct in the data source configuration. +1. Ensure the user is mapped to the database with appropriate permissions. +1. Grant at least `SELECT` permission on the required tables: + + ```sql + USE [your_database] + GRANT SELECT ON dbo.YourTable TO [your_user] + ``` + +1. Check that the user doesn't have any conflicting permissions from the public role. + +### Windows Authentication (Kerberos) issues + +**Error message:** "Kerberos authentication failed" or "Cannot initialize Kerberos" + +**Cause:** Kerberos configuration is incorrect or incomplete. + +**Solution:** + +1. Verify that the Kerberos configuration file (`krb5.conf`) path is correct in the data source settings. +1. For keytab authentication, ensure the keytab file exists and is readable by Grafana. +1. Check that the realm and KDC settings are correct in the Kerberos configuration. +1. Verify DNS is correctly resolving the KDC servers. +1. Ensure the service principal name (SPN) is registered for the SQL Server instance. + +{{< admonition type="note" >}} +Kerberos authentication is not supported in Grafana Cloud. +{{< /admonition >}} + +### Azure Entra ID authentication errors + +**Error message:** "AADSTS error codes" or "Azure AD authentication failed" + +**Cause:** Azure Entra ID (formerly Azure AD) authentication is misconfigured. + +**Solution:** + +1. For **App Registration** authentication: + - Verify the tenant ID, client ID, and client secret are correct. + - Ensure the app registration has been added as a user in the Azure SQL database. + - Check that the client secret hasn't expired. + +1. For **Managed Identity** authentication: + - Verify `managed_identity_enabled = true` is set in the Grafana server configuration. + - Ensure the managed identity has been added to the Azure SQL database. + - Confirm the Azure resource hosting Grafana has managed identity enabled. + +1. For **Current User** authentication: + - Ensure `user_identity_enabled = true` is set in the Grafana server configuration. + - Verify the app registration is configured to issue both Access Tokens and ID Tokens. + - Check that the required API permissions are configured (`user_impersonation` for Azure SQL). + +For detailed Azure authentication configuration, refer to [Configure the Microsoft SQL Server data source](ref:configure-mssql-data-source). + +## Query errors + +These errors occur when there are issues with query syntax or configuration. + +### Time column not found or invalid + +**Error message:** "Could not find time column" or time series visualization shows no data + +**Cause:** The query doesn't return a properly formatted `time` column for time series visualization. + +**Solution:** + +1. Ensure your query includes a column named `time` when using the **Time series** format. +1. Use the `$__time()` macro to rename your date column: `$__time(your_date_column)`. +1. Verify the time column is of a valid SQL date/time type (`datetime`, `datetime2`, `date`) or contains Unix epoch values. +1. Ensure the result set is sorted by the time column using `ORDER BY`. + +### Macro expansion errors + +**Error message:** "Error parsing query" or macros appear unexpanded in the query + +**Cause:** Grafana macros are being used incorrectly. + +**Solution:** + +1. Verify macro syntax: use `$__timeFilter(column)` not `$_timeFilter(column)`. +1. Macros don't work inside stored proceduresโ€”use explicit date parameters instead. +1. Check that the column name passed to macros exists in your table. +1. View the expanded query by clicking **Generated SQL** after running the query to debug macro expansion. + +### Timezone and time shift issues + +**Cause:** Time series data appears shifted or doesn't align with expected times. + +**Solution:** + +1. Store timestamps in UTC in your database to avoid timezone issues. +1. Time macros (`$__time`, `$__timeFilter`, etc.) always expand to UTC values. +1. If your timestamps are stored in local time, convert them to UTC in your query: + + ```sql + SELECT + your_datetime_column AT TIME ZONE 'Your Local Timezone' AT TIME ZONE 'UTC' AS time, + value + FROM your_table + ``` + +1. Don't pass timezone parameters to time macrosโ€”they're not supported. + +### Query returns too many rows + +**Error message:** "Result set too large" or browser becomes unresponsive + +**Cause:** The query returns more data than can be efficiently processed. + +**Solution:** + +1. Add time filters using `$__timeFilter(column)` to limit data to the dashboard time range. +1. Use aggregations (`AVG`, `SUM`, `COUNT`) with `GROUP BY` instead of returning raw rows. +1. Add a `TOP` clause to limit results: `SELECT TOP 1000 ...`. +1. Use the `$__timeGroup()` macro to aggregate data into time intervals. + +### Stored procedure returns no data + +**Cause:** Stored procedure output isn't being captured correctly. + +**Solution:** + +1. Ensure the stored procedure uses `SELECT` statements, not just variable assignments. +1. Remove `SET NOCOUNT ON` if present, or ensure it's followed by a `SELECT` statement. +1. Verify the stored procedure parameters are being passed correctly. +1. Test the stored procedure directly in SQL Server Management Studio with the same parameters. + +For more information on using stored procedures, refer to the [query editor documentation](ref:mssql-query-editor). + +## Performance issues + +These issues relate to slow queries or high resource usage. + +### Slow query execution + +**Cause:** Queries take a long time to execute. + +**Solution:** + +1. Reduce the dashboard time range to limit data volume. +1. Add indexes to columns used in `WHERE` clauses and time filters. +1. Use aggregations instead of returning individual rows. +1. Increase the **Min time interval** setting to reduce the number of data points. +1. Review the query execution plan in SQL Server Management Studio to identify bottlenecks. + +### Connection pool exhaustion + +**Error message:** "Too many connections" or "Connection pool exhausted" + +**Cause:** Too many concurrent connections to the database. + +**Solution:** + +1. Increase the **Max open** connection limit in the data source configuration. +1. Enable **Auto max idle** to automatically manage idle connections. +1. Reduce the number of panels querying the same data source simultaneously. +1. Check for long-running queries that might be holding connections. + +## Other common issues + +The following issues don't produce specific error messages but are commonly encountered. + +### System databases appear in queries + +**Cause:** Queries accidentally access system databases. + +**Solution:** + +1. The query editor automatically excludes `tempdb`, `model`, `msdb`, and `master` from the database dropdown. +1. Always specify the database in your data source configuration to restrict access. +1. Ensure the database user only has permissions on the intended database. + +### Template variable queries fail + +**Cause:** Variable queries return unexpected results or errors. + +**Solution:** + +1. Verify the variable query syntax is valid SQL that returns a single column. +1. Check that the data source connection is working. +1. Ensure the user has permission to access the tables referenced in the variable query. +1. Test the query in the query editor before using it as a variable query. + +### Data appears incorrect or misaligned + +**Cause:** Data formatting or type conversion issues. + +**Solution:** + +1. Use explicit column aliases to ensure consistent naming: `SELECT value AS metric`. +1. Verify numeric columns are actually numeric types, not strings. +1. Check for `NULL` values that might affect aggregations. +1. Use the `FILL` option in `$__timeGroup()` macro to handle missing data points. + +## Get additional help + +If you continue to experience issues after following this troubleshooting guide: + +1. Check the [Grafana community forums](https://community.grafana.com/) for similar issues. +1. Review the [Grafana GitHub issues](https://github.com/grafana/grafana/issues) for known bugs. +1. Enable debug logging in Grafana to capture detailed error information. +1. Check SQL Server logs for additional error details. +1. Contact Grafana Support if you're an Enterprise or Cloud customer. + +When reporting issues, include: + +- Grafana version +- SQL Server version +- Error messages (redact sensitive information) +- Steps to reproduce +- Relevant query examples (redact sensitive data) diff --git a/packages/grafana-api-clients/src/clients/rtkq/dashboard/v0alpha1/endpoints.gen.ts b/packages/grafana-api-clients/src/clients/rtkq/dashboard/v0alpha1/endpoints.gen.ts index 326b53ccedd..9605b8e9355 100644 --- a/packages/grafana-api-clients/src/clients/rtkq/dashboard/v0alpha1/endpoints.gen.ts +++ b/packages/grafana-api-clients/src/clients/rtkq/dashboard/v0alpha1/endpoints.gen.ts @@ -246,6 +246,8 @@ const injectedRtkApi = api facetLimit: queryArg.facetLimit, tags: queryArg.tags, libraryPanel: queryArg.libraryPanel, + panelType: queryArg.panelType, + dataSourceType: queryArg.dataSourceType, permission: queryArg.permission, sort: queryArg.sort, limit: queryArg.limit, @@ -674,6 +676,10 @@ export type SearchDashboardsAndFoldersApiArg = { tags?: string[]; /** find dashboards that reference a given libraryPanel */ libraryPanel?: string; + /** find dashboards using panels of a given plugin type */ + panelType?: string; + /** find dashboards using datasources of a given plugin type */ + dataSourceType?: string; /** permission needed for the resource (view, edit, admin) */ permission?: 'view' | 'edit' | 'admin'; /** sortable field */ diff --git a/packages/grafana-api-clients/src/clients/rtkq/provisioning/v0alpha1/endpoints.gen.ts b/packages/grafana-api-clients/src/clients/rtkq/provisioning/v0alpha1/endpoints.gen.ts index b6519295c66..40d4299a1a0 100644 --- a/packages/grafana-api-clients/src/clients/rtkq/provisioning/v0alpha1/endpoints.gen.ts +++ b/packages/grafana-api-clients/src/clients/rtkq/provisioning/v0alpha1/endpoints.gen.ts @@ -1452,7 +1452,7 @@ export type ConnectionSecure = { /** PrivateKey is the reference to the private key used for GitHub App authentication. This value is stored securely and cannot be read back */ privateKey?: InlineSecureValue; /** Token is the reference of the token used to act as the Connection. This value is stored securely and cannot be read back */ - webhook?: InlineSecureValue; + token?: InlineSecureValue; }; export type BitbucketConnectionConfig = { /** App client ID */ diff --git a/packages/grafana-data/src/field/fieldOverrides.test.ts b/packages/grafana-data/src/field/fieldOverrides.test.ts index 767da439543..b0aa271be3f 100644 --- a/packages/grafana-data/src/field/fieldOverrides.test.ts +++ b/packages/grafana-data/src/field/fieldOverrides.test.ts @@ -9,6 +9,7 @@ import { FieldColorModeId } from '../types/fieldColor'; import { FieldConfigPropertyItem, FieldConfigSource } from '../types/fieldOverrides'; import { InterpolateFunction } from '../types/panel'; import { ThresholdsMode } from '../types/thresholds'; +import { MappingType } from '../types/valueMapping'; import { Registry } from '../utils/Registry'; import { locationUtil } from '../utils/location'; import { mockStandardProperties } from '../utils/tests/mockStandardProperties'; @@ -999,6 +1000,45 @@ describe('setDynamicConfigValue', () => { expect(config.custom.property3).toEqual({}); expect(config.displayName).toBeUndefined(); }); + + it('works correctly with multiple value mappings in the same override', () => { + const config: FieldConfig = { + mappings: [{ type: MappingType.ValueToText, options: { existing: { text: 'existing' } } }], + }; + + setDynamicConfigValue( + config, + { + id: 'mappings', + value: [{ type: MappingType.ValueToText, options: { first: { text: 'first' } } }], + }, + { + fieldConfigRegistry: customFieldRegistry, + data: [], + field: { type: FieldType.number } as Field, + dataFrameIndex: 0, + } + ); + + setDynamicConfigValue( + config, + { + id: 'mappings', + value: [{ type: MappingType.ValueToText, options: { second: { text: 'second' } } }], + }, + { + fieldConfigRegistry: customFieldRegistry, + data: [], + field: { type: FieldType.number } as Field, + dataFrameIndex: 0, + } + ); + + expect(config.mappings).toHaveLength(3); + expect(config.mappings![0]).toEqual({ type: MappingType.ValueToText, options: { existing: { text: 'existing' } } }); + expect(config.mappings![1]).toEqual({ type: MappingType.ValueToText, options: { first: { text: 'first' } } }); + expect(config.mappings![2]).toEqual({ type: MappingType.ValueToText, options: { second: { text: 'second' } } }); + }); }); describe('getLinksSupplier', () => { diff --git a/packages/grafana-data/src/field/fieldOverrides.ts b/packages/grafana-data/src/field/fieldOverrides.ts index 8b345036c64..bf56f811106 100644 --- a/packages/grafana-data/src/field/fieldOverrides.ts +++ b/packages/grafana-data/src/field/fieldOverrides.ts @@ -341,7 +341,7 @@ export function setDynamicConfigValue(config: FieldConfig, value: DynamicConfigV return; } - const val = item.process(value.value, context, item.settings); + let val = item.process(value.value, context, item.settings); const remove = val === undefined || val === null; @@ -352,6 +352,15 @@ export function setDynamicConfigValue(config: FieldConfig, value: DynamicConfigV unset(config, item.path); } } else { + // Merge arrays (e.g. mappings) when multiple overrides target the same field + if (Array.isArray(val)) { + const existingValue = item.isCustom ? get(config.custom, item.path) : get(config, item.path); + + if (Array.isArray(existingValue)) { + val = [...existingValue, ...val]; + } + } + if (item.isCustom) { if (!config.custom) { config.custom = {}; diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index ef1c6803962..3e0439a872f 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -531,10 +531,6 @@ export interface FeatureToggles { */ dashboardTemplates?: boolean; /** - * Sets the logs table as default visualisation in logs explore - */ - logsExploreTableDefaultVisualization?: boolean; - /** * Enables the new alert list view design */ alertingListViewV2?: boolean; @@ -661,10 +657,6 @@ export interface FeatureToggles { */ rolePickerDrawer?: boolean; /** - * Enable unified storage search - */ - unifiedStorageSearch?: boolean; - /** * Enable sprinkles on unified storage search */ unifiedStorageSearchSprinkles?: boolean; @@ -707,10 +699,6 @@ export interface FeatureToggles { */ passwordlessMagicLinkAuthentication?: boolean; /** - * Display Related Logs in Grafana Metrics Drilldown - */ - exploreMetricsRelatedLogs?: boolean; - /** * Adds support for quotes and special characters in label values for Prometheus queries */ prometheusSpecialCharsInLabelValues?: boolean; diff --git a/packages/grafana-data/src/types/icon.ts b/packages/grafana-data/src/types/icon.ts index 34e672b66f5..3dc7215a2cf 100644 --- a/packages/grafana-data/src/types/icon.ts +++ b/packages/grafana-data/src/types/icon.ts @@ -52,6 +52,7 @@ export const availableIconsIndex = { bookmark: true, 'book-open': true, 'brackets-curly': true, + brain: true, 'browser-alt': true, bug: true, building: true, diff --git a/packages/grafana-i18n/package.json b/packages/grafana-i18n/package.json index 36a9faa541b..93dd9837897 100644 --- a/packages/grafana-i18n/package.json +++ b/packages/grafana-i18n/package.json @@ -29,7 +29,6 @@ "@grafana-app/source": "./src/internal/index.ts" }, "./eslint-plugin": { - "@grafana-app/source": "./src/eslint/index.cjs", "types": "./src/eslint/index.d.ts", "default": "./src/eslint/index.cjs" } diff --git a/pkg/registry/apis/dashboard/search.go b/pkg/registry/apis/dashboard/search.go index 08a943b8da6..8572dae8295 100644 --- a/pkg/registry/apis/dashboard/search.go +++ b/pkg/registry/apis/dashboard/search.go @@ -142,6 +142,24 @@ func (s *SearchHandler) GetAPIRoutes(defs map[string]common.OpenAPIDefinition) * Schema: spec.StringProperty(), }, }, + { + ParameterProps: spec3.ParameterProps{ + Name: "panelType", + In: "query", + Description: "find dashboards using panels of a given plugin type", + Required: false, + Schema: spec.StringProperty(), + }, + }, + { + ParameterProps: spec3.ParameterProps{ + Name: "dataSourceType", + In: "query", + Description: "find dashboards using datasources of a given plugin type", + Required: false, + Schema: spec.StringProperty(), + }, + }, { ParameterProps: spec3.ParameterProps{ Name: "permission", @@ -430,14 +448,11 @@ func convertHttpSearchRequestToResourceSearchRequest(queryParams url.Values, use } } - // The facet term fields + // Apply facet terms if facets, ok := queryParams["facet"]; ok { if queryParams.Has("facetLimit") { if parsed, err := strconv.Atoi(queryParams.Get("facetLimit")); err == nil && parsed > 0 { - facetLimit = parsed - if facetLimit > 1000 { - facetLimit = 1000 - } + facetLimit = min(parsed, 1000) } } searchRequest.Facet = make(map[string]*resourcepb.ResourceSearchRequest_Facet) @@ -449,21 +464,35 @@ func convertHttpSearchRequestToResourceSearchRequest(queryParams url.Values, use } } - // The tags filter - if tags, ok := queryParams["tag"]; ok { + if v, ok := queryParams["tag"]; ok { searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{ Key: "tags", Operator: "=", - Values: tags, + Values: v, }) } - // The libraryPanel filter - if libraryPanel, ok := queryParams["libraryPanel"]; ok { + if v, ok := queryParams["panelType"]; ok { + searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{ + Key: resource.SEARCH_FIELD_PREFIX + builders.DASHBOARD_PANEL_TYPES, + Operator: "=", + Values: v, + }) + } + + if v, ok := queryParams["dataSourceType"]; ok { + searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{ + Key: resource.SEARCH_FIELD_PREFIX + builders.DASHBOARD_DS_TYPES, + Operator: "=", + Values: v, + }) + } + + if v, ok := queryParams["libraryPanel"]; ok { searchRequest.Options.Fields = append(searchRequest.Options.Fields, &resourcepb.Requirement{ Key: builders.DASHBOARD_LIBRARY_PANEL_REFERENCE, Operator: "=", - Values: libraryPanel, + Values: v, }) } diff --git a/pkg/registry/apis/datasource/plugincontext.go b/pkg/registry/apis/datasource/plugincontext.go index a0525030dcb..27479f60675 100644 --- a/pkg/registry/apis/datasource/plugincontext.go +++ b/pkg/registry/apis/datasource/plugincontext.go @@ -71,7 +71,6 @@ type cachingDatasourceProvider struct { } func (q *cachingDatasourceProvider) GetDatasourceProvider(pluginJson plugins.JSONData) PluginDatasourceProvider { - group, _ := plugins.GetDatasourceGroupNameFromPluginID(pluginJson.ID) return &scopedDatasourceProvider{ plugin: pluginJson, dsService: q.dsService, @@ -81,7 +80,7 @@ func (q *cachingDatasourceProvider) GetDatasourceProvider(pluginJson plugins.JSO mapper: q.converter.mapper, plugin: pluginJson.ID, alias: pluginJson.AliasIDs, - group: group, + group: pluginJson.ID, }, } } diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index 9222c020681..e23a3c1ab2a 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -37,6 +37,11 @@ var ( _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) ) +type DataSourceAPIBuilderConfig struct { + LoadQueryTypes bool + UseDualWriter bool +} + // DataSourceAPIBuilder is used just so wire has something unique to return type DataSourceAPIBuilder struct { datasourceResourceInfo utils.ResourceInfo @@ -46,7 +51,7 @@ type DataSourceAPIBuilder struct { contextProvider PluginContextWrapper accessControl accesscontrol.AccessControl queryTypes *queryV0.QueryTypeDefinitionList - configCrudUseNewApis bool + cfg DataSourceAPIBuilderConfig dataSourceCRUDMetric *prometheus.HistogramVec } @@ -89,20 +94,24 @@ func RegisterAPIService( return nil, fmt.Errorf("plugin client is not a PluginClient: %T", pluginClient) } + groupName := pluginJSON.ID + ".datasource.grafana.app" builder, err = NewDataSourceAPIBuilder( + groupName, pluginJSON, client, datasources.GetDatasourceProvider(pluginJSON), contextProvider, accessControl, - //nolint:staticcheck // not yet migrated to OpenFeature - features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes), - //nolint:staticcheck // not yet migrated to OpenFeature - features.IsEnabledGlobally(featuremgmt.FlagQueryServiceWithConnections), + DataSourceAPIBuilderConfig{ + //nolint:staticcheck // not yet migrated to OpenFeature + LoadQueryTypes: features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes), + UseDualWriter: false, + }, ) if err != nil { return nil, err } + builder.SetDataSourceCRUDMetrics(dataSourceCRUDMetric) apiRegistrar.RegisterAPI(builder) @@ -120,31 +129,27 @@ type PluginClient interface { } func NewDataSourceAPIBuilder( + groupName string, plugin plugins.JSONData, client PluginClient, datasources PluginDatasourceProvider, contextProvider PluginContextWrapper, accessControl accesscontrol.AccessControl, - loadQueryTypes bool, - configCrudUseNewApis bool, + cfg DataSourceAPIBuilderConfig, ) (*DataSourceAPIBuilder, error) { - group, err := plugins.GetDatasourceGroupNameFromPluginID(plugin.ID) - if err != nil { - return nil, err - } - builder := &DataSourceAPIBuilder{ - datasourceResourceInfo: datasourceV0.DataSourceResourceInfo.WithGroupAndShortName(group, plugin.ID), + datasourceResourceInfo: datasourceV0.DataSourceResourceInfo.WithGroupAndShortName(groupName, plugin.ID), pluginJSON: plugin, client: client, datasources: datasources, contextProvider: contextProvider, accessControl: accessControl, - configCrudUseNewApis: configCrudUseNewApis, + cfg: cfg, } - if loadQueryTypes { + var err error + if cfg.LoadQueryTypes { // In the future, this will somehow come from the plugin - builder.queryTypes, err = getHardcodedQueryTypes(group) + builder.queryTypes, err = getHardcodedQueryTypes(groupName) } return builder, err } @@ -154,9 +159,9 @@ func getHardcodedQueryTypes(group string) (*queryV0.QueryTypeDefinitionList, err var err error var raw json.RawMessage switch group { - case "testdata.datasource.grafana.app": + case "testdata.datasource.grafana.app", "grafana-testdata-datasource": raw, err = kinds.QueryTypeDefinitionListJSON() - case "prometheus.datasource.grafana.app": + case "prometheus.datasource.grafana.app", "prometheus": raw, err = models.QueryTypeDefinitionListJSON() } if err != nil { @@ -233,7 +238,7 @@ func (b *DataSourceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver storage["connections"] = &noopREST{} // hidden from openapi storage["connections/query"] = storage[ds.StoragePath("query")] // deprecated in openapi - if b.configCrudUseNewApis { + if b.cfg.UseDualWriter { legacyStore := &legacyStorage{ datasources: b.datasources, resourceInfo: &ds, diff --git a/pkg/registry/apis/provisioning/extras/register.go b/pkg/registry/apis/provisioning/extras/register.go index caa3c2a2fa5..43c7064173f 100644 --- a/pkg/registry/apis/provisioning/extras/register.go +++ b/pkg/registry/apis/provisioning/extras/register.go @@ -2,6 +2,8 @@ package extras import ( apisprovisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + "github.com/grafana/grafana/apps/provisioning/pkg/connection" + ghconnection "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" "github.com/grafana/grafana/apps/provisioning/pkg/repository" "github.com/grafana/grafana/apps/provisioning/pkg/repository/git" "github.com/grafana/grafana/apps/provisioning/pkg/repository/github" @@ -42,6 +44,15 @@ func ProvideProvisioningOSSRepositoryExtras( } } +func ProvideProvisioningOSSConnectionExtras( + _ *setting.Cfg, + ghFactory ghconnection.GithubFactory, +) []connection.Extra { + return []connection.Extra{ + ghconnection.Extra(ghFactory), + } +} + func ProvideExtraWorkers(pullRequestWorker *pullrequest.PullRequestWorker) []jobs.Worker { return []jobs.Worker{pullRequestWorker} } @@ -54,3 +65,12 @@ func ProvideFactoryFromConfig(cfg *setting.Cfg, extras []repository.Extra) (repo return repository.ProvideFactory(enabledTypes, extras) } + +func ProvideConnectionFactoryFromConfig(cfg *setting.Cfg, extras []connection.Extra) (connection.Factory, error) { + enabledTypes := make(map[apisprovisioning.ConnectionType]struct{}, len(cfg.ProvisioningRepositoryTypes)) + for _, e := range cfg.ProvisioningRepositoryTypes { + enabledTypes[apisprovisioning.ConnectionType(e)] = struct{}{} + } + + return connection.ProvideFactory(enabledTypes, extras) +} diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index e54a8c2fc28..901bc829c97 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -30,7 +30,7 @@ import ( provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" "github.com/grafana/grafana/apps/provisioning/pkg/auth" - connectionvalidation "github.com/grafana/grafana/apps/provisioning/pkg/connection" + "github.com/grafana/grafana/apps/provisioning/pkg/connection" appcontroller "github.com/grafana/grafana/apps/provisioning/pkg/controller" clientset "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned" client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1" @@ -105,20 +105,21 @@ type APIBuilder struct { jobs.Queue jobs.Store } - jobHistoryConfig *JobHistoryConfig - jobHistoryLoki *jobs.LokiJobHistory - resourceLister resources.ResourceLister - dashboardAccess legacy.MigrationDashboardAccessor - unified resource.ResourceClient - repoFactory repository.Factory - client client.ProvisioningV0alpha1Interface - access auth.AccessChecker - accessWithAdmin auth.AccessChecker - accessWithEditor auth.AccessChecker - accessWithViewer auth.AccessChecker - statusPatcher *appcontroller.RepositoryStatusPatcher - healthChecker *controller.HealthChecker - validator repository.RepositoryValidator + jobHistoryConfig *JobHistoryConfig + jobHistoryLoki *jobs.LokiJobHistory + resourceLister resources.ResourceLister + dashboardAccess legacy.MigrationDashboardAccessor + unified resource.ResourceClient + repoFactory repository.Factory + connectionFactory connection.Factory + client client.ProvisioningV0alpha1Interface + access auth.AccessChecker + accessWithAdmin auth.AccessChecker + accessWithEditor auth.AccessChecker + accessWithViewer auth.AccessChecker + statusPatcher *appcontroller.RepositoryStatusPatcher + healthChecker *controller.HealthChecker + repoValidator repository.RepositoryValidator // Extras provides additional functionality to the API. extras []Extra extraWorkers []jobs.Worker @@ -133,6 +134,7 @@ type APIBuilder struct { func NewAPIBuilder( onlyApiServer bool, repoFactory repository.Factory, + connectionFactory connection.Factory, features featuremgmt.FeatureToggles, unified resource.ResourceClient, configProvider apiserver.RestConfigProvider, @@ -176,6 +178,7 @@ func NewAPIBuilder( usageStats: usageStats, features: features, repoFactory: repoFactory, + connectionFactory: connectionFactory, clients: clients, parsers: parsers, repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister), @@ -192,7 +195,7 @@ func NewAPIBuilder( allowedTargets: allowedTargets, allowImageRendering: allowImageRendering, registry: registry, - validator: repository.NewValidator(minSyncInterval, allowedTargets, allowImageRendering), + repoValidator: repository.NewValidator(minSyncInterval, allowedTargets, allowImageRendering), useExclusivelyAccessCheckerForAuthz: useExclusivelyAccessCheckerForAuthz, } @@ -253,6 +256,7 @@ func RegisterAPIService( extraBuilders []ExtraBuilder, extraWorkers []jobs.Worker, repoFactory repository.Factory, + connectionFactory connection.Factory, ) (*APIBuilder, error) { //nolint:staticcheck // not yet migrated to OpenFeature if !features.IsEnabledGlobally(featuremgmt.FlagProvisioning) { @@ -271,6 +275,7 @@ func RegisterAPIService( builder := NewAPIBuilder( cfg.DisableControllers, repoFactory, + connectionFactory, features, client, configProvider, @@ -641,7 +646,7 @@ func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupI storage[provisioning.ConnectionResourceInfo.StoragePath("repositories")] = NewConnectionRepositoriesConnector() // TODO: Add some logic so that the connectors can registered themselves and we don't have logic all over the place - storage[provisioning.RepositoryResourceInfo.StoragePath("test")] = NewTestConnector(b, repository.NewRepositoryTesterWithExistingChecker(repository.NewSimpleRepositoryTester(b.validator), b.VerifyAgainstExistingRepositories)) + storage[provisioning.RepositoryResourceInfo.StoragePath("test")] = NewTestConnector(b, repository.NewRepositoryTesterWithExistingChecker(repository.NewSimpleRepositoryTester(b.repoValidator), b.VerifyAgainstExistingRepositories)) storage[provisioning.RepositoryResourceInfo.StoragePath("files")] = NewFilesConnector(b, b.parsers, b.clients, b.accessWithAdmin) storage[provisioning.RepositoryResourceInfo.StoragePath("refs")] = NewRefsConnector(b) storage[provisioning.RepositoryResourceInfo.StoragePath("resources")] = &listConnector{ @@ -682,10 +687,15 @@ func (b *APIBuilder) Mutate(ctx context.Context, a admission.Attributes, o admis if ok { return nil } - // TODO: complete this as part of https://github.com/grafana/git-ui-sync-project/issues/700 + c, ok := obj.(*provisioning.Connection) if ok { - return connectionvalidation.MutateConnection(c) + conn, err := b.asConnection(ctx, c, nil) + if err != nil { + return err + } + + return conn.Mutate(ctx) } r, ok := obj.(*provisioning.Repository) @@ -736,9 +746,15 @@ func (b *APIBuilder) Validate(ctx context.Context, a admission.Attributes, o adm return nil } - connection, ok := obj.(*provisioning.Connection) + // Validate connections + c, ok := obj.(*provisioning.Connection) if ok { - return connectionvalidation.ValidateConnection(connection) + conn, err := b.asConnection(ctx, c, a.GetOldObject()) + if err != nil { + return err + } + + return conn.Validate(ctx) } // Validate Jobs @@ -758,7 +774,7 @@ func (b *APIBuilder) Validate(ctx context.Context, a admission.Attributes, o adm // the only time to add configuration checks here is if you need to compare // the incoming change to the current configuration isCreate := a.GetOperation() == admission.Create - list := b.validator.ValidateRepository(repo, isCreate) + list := b.repoValidator.ValidateRepository(repo, isCreate) cfg := repo.Config() if a.GetOperation() == admission.Update { @@ -831,7 +847,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH } b.statusPatcher = appcontroller.NewRepositoryStatusPatcher(b.GetClient()) - b.healthChecker = controller.NewHealthChecker(b.statusPatcher, b.registry, repository.NewSimpleRepositoryTester(b.validator)) + b.healthChecker = controller.NewHealthChecker(b.statusPatcher, b.registry, repository.NewSimpleRepositoryTester(b.repoValidator)) // if running solely CRUD, skip the rest of the setup if b.onlyApiServer { @@ -1449,6 +1465,35 @@ func (b *APIBuilder) asRepository(ctx context.Context, obj runtime.Object, old r return b.repoFactory.Build(ctx, r) } +func (b *APIBuilder) asConnection(ctx context.Context, obj runtime.Object, old runtime.Object) (connection.Connection, error) { + if obj == nil { + return nil, fmt.Errorf("missing connection object") + } + + c, ok := obj.(*provisioning.Connection) + if !ok { + return nil, fmt.Errorf("expected connection object") + } + + // Copy previous values if they exist + if old != nil { + o, ok := old.(*provisioning.Connection) + if ok && !o.Secure.IsZero() { + if c.Secure.PrivateKey.IsZero() { + c.Secure.PrivateKey = o.Secure.PrivateKey + } + if c.Secure.Token.IsZero() { + c.Secure.Token = o.Secure.Token + } + if c.Secure.ClientSecret.IsZero() { + c.Secure.ClientSecret = o.Secure.ClientSecret + } + } + } + + return b.connectionFactory.Build(ctx, c) +} + func getJSONResponse(ref string) *spec3.Responses { return &spec3.Responses{ ResponsesProps: spec3.ResponsesProps{ diff --git a/pkg/registry/apis/provisioning/register_validate_test.go b/pkg/registry/apis/provisioning/register_validate_test.go index 18b366e4de8..f612d3a408c 100644 --- a/pkg/registry/apis/provisioning/register_validate_test.go +++ b/pkg/registry/apis/provisioning/register_validate_test.go @@ -28,7 +28,7 @@ func TestAPIBuilderValidate(t *testing.T) { repoFactory: factory, allowedTargets: []v0alpha1.SyncTargetType{v0alpha1.SyncTargetTypeFolder}, allowImageRendering: false, - validator: validator, + repoValidator: validator, } t.Run("min sync interval is less than 10 seconds", func(t *testing.T) { diff --git a/pkg/registry/apis/wireset.go b/pkg/registry/apis/wireset.go index d95f01bef36..296b7d25b59 100644 --- a/pkg/registry/apis/wireset.go +++ b/pkg/registry/apis/wireset.go @@ -44,6 +44,7 @@ var provisioningExtras = wire.NewSet( pullrequest.ProvidePullRequestWorker, webhooks.ProvideWebhooksWithImages, extras.ProvideFactoryFromConfig, + extras.ProvideConnectionFactoryFromConfig, extras.ProvideProvisioningExtraAPIs, extras.ProvideExtraWorkers, ) diff --git a/pkg/server/test_env.go b/pkg/server/test_env.go index 76fa96a75c7..57c7e87bbb9 100644 --- a/pkg/server/test_env.go +++ b/pkg/server/test_env.go @@ -3,6 +3,7 @@ package server import ( "github.com/stretchr/testify/mock" + githubconnection "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" "github.com/grafana/grafana/apps/provisioning/pkg/repository/github" "github.com/grafana/grafana/apps/secret/pkg/decrypt" "github.com/grafana/grafana/pkg/infra/db" @@ -34,24 +35,26 @@ func ProvideTestEnv( featureMgmt featuremgmt.FeatureToggles, resourceClient resource.ResourceClient, idService auth.IDService, - githubFactory *github.Factory, + githubRepoFactory *github.Factory, + githubConnectionFactory githubconnection.GithubFactory, decryptService decrypt.DecryptService, ) (*TestEnv, error) { return &TestEnv{ - TestingT: testingT, - Server: server, - SQLStore: db, - Cfg: cfg, - NotificationService: ns, - GRPCServer: grpcServer, - PluginRegistry: pluginRegistry, - HTTPClientProvider: httpClientProvider, - OAuthTokenService: oAuthTokenService, - FeatureToggles: featureMgmt, - ResourceClient: resourceClient, - IDService: idService, - GitHubFactory: githubFactory, - DecryptService: decryptService, + TestingT: testingT, + Server: server, + SQLStore: db, + Cfg: cfg, + NotificationService: ns, + GRPCServer: grpcServer, + PluginRegistry: pluginRegistry, + HTTPClientProvider: httpClientProvider, + OAuthTokenService: oAuthTokenService, + FeatureToggles: featureMgmt, + ResourceClient: resourceClient, + IDService: idService, + GithubRepoFactory: githubRepoFactory, + GithubConnectionFactory: githubConnectionFactory, + DecryptService: decryptService, }, nil } @@ -60,18 +63,19 @@ type TestEnv struct { mock.TestingT Cleanup(func()) } - Server *Server - SQLStore db.DB - Cfg *setting.Cfg - NotificationService *notifications.NotificationServiceMock - GRPCServer grpcserver.Provider - PluginRegistry registry.Service - HTTPClientProvider httpclient.Provider - OAuthTokenService *oauthtokentest.Service - RequestMiddleware web.Middleware - FeatureToggles featuremgmt.FeatureToggles - ResourceClient resource.ResourceClient - IDService auth.IDService - GitHubFactory *github.Factory - DecryptService decrypt.DecryptService + Server *Server + SQLStore db.DB + Cfg *setting.Cfg + NotificationService *notifications.NotificationServiceMock + GRPCServer grpcserver.Provider + PluginRegistry registry.Service + HTTPClientProvider httpclient.Provider + OAuthTokenService *oauthtokentest.Service + RequestMiddleware web.Middleware + FeatureToggles featuremgmt.FeatureToggles + ResourceClient resource.ResourceClient + IDService auth.IDService + GithubRepoFactory *github.Factory + GithubConnectionFactory githubconnection.GithubFactory + DecryptService decrypt.DecryptService } diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 141079b1513..399794e9787 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -15,6 +15,7 @@ import ( "go.opentelemetry.io/otel/trace" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + ghconnection "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" "github.com/grafana/grafana/apps/provisioning/pkg/repository/github" "github.com/grafana/grafana/pkg/api" "github.com/grafana/grafana/pkg/api/avatar" @@ -297,6 +298,7 @@ var wireBasicSet = wire.NewSet( notifications.ProvideService, notifications.ProvideSmtpService, github.ProvideFactory, + ghconnection.ProvideFactory, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index 65e6ebeb36e..218e9fabc36 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -10,6 +10,7 @@ import ( "github.com/google/wire" httpclient2 "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/apps/advisor/pkg/app/checkregistry" + github2 "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" "github.com/grafana/grafana/apps/provisioning/pkg/repository/github" "github.com/grafana/grafana/pkg/api" "github.com/grafana/grafana/pkg/api/avatar" @@ -914,7 +915,13 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api if err != nil { return nil, err } - provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, migrationDashboardAccessor, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory) + githubFactory := github2.ProvideFactory() + v7 := extras.ProvideProvisioningOSSConnectionExtras(cfg, githubFactory) + connectionFactory, err := extras.ProvideConnectionFactoryFromConfig(cfg, v7) + if err != nil { + return nil, err + } + provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, migrationDashboardAccessor, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory, connectionFactory) if err != nil { return nil, err } @@ -1576,7 +1583,13 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } - provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, migrationDashboardAccessor, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory) + githubFactory := github2.ProvideFactory() + v7 := extras.ProvideProvisioningOSSConnectionExtras(cfg, githubFactory) + connectionFactory, err := extras.ProvideConnectionFactoryFromConfig(cfg, v7) + if err != nil { + return nil, err + } + provisioningAPIBuilder, err := provisioning2.RegisterAPIService(cfg, featureToggles, apiserverService, registerer, resourceClient, eventualRestConfigProvider, accessClient, migrationDashboardAccessor, dualwriteService, usageStats, tracingService, v3, v4, repositoryFactory, connectionFactory) if err != nil { return nil, err } @@ -1610,7 +1623,7 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac if err != nil { return nil, err } - testEnv, err := ProvideTestEnv(testingT, server, sqlStore, cfg, notificationServiceMock, grpcserverProvider, inMemory, httpclientProvider, oauthtokentestService, featureToggles, resourceClient, idimplService, factory, decryptService) + testEnv, err := ProvideTestEnv(testingT, server, sqlStore, cfg, notificationServiceMock, grpcserverProvider, inMemory, httpclientProvider, oauthtokentestService, featureToggles, resourceClient, idimplService, factory, githubFactory, decryptService) if err != nil { return nil, err } @@ -1800,7 +1813,7 @@ var withOTelSet = wire.NewSet( otelTracer, grpcserver.ProvideService, interceptors.ProvideAuthenticator, ) -var wireBasicSet = wire.NewSet(annotationsimpl.ProvideService, wire.Bind(new(annotations.Repository), new(*annotationsimpl.RepositoryImpl)), New, api.ProvideHTTPServer, query.ProvideService, wire.Bind(new(query.Service), new(*query.ServiceImpl)), bus.ProvideBus, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), rendering.ProvideService, wire.Bind(new(rendering.Service), new(*rendering.RenderingService)), routing.ProvideRegister, wire.Bind(new(routing.RouteRegister), new(*routing.RouteRegisterImpl)), hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, bundleregistry.ProvideService, wire.Bind(new(supportbundles.Service), new(*bundleregistry.Service)), updatemanager.ProvideGrafanaService, updatemanager.ProvidePluginsService, service.ProvideService, wire.Bind(new(usagestats.Service), new(*service.UsageStats)), validator3.ProvideService, provisioning.ProvideStubProvisioningService, legacy.ProvideMigratorDashboardAccessor, migrations2.ProvideUnifiedMigrator, pluginsintegration.WireSet, dashboards.ProvideFileStoreManager, wire.Bind(new(dashboards.FileStore), new(*dashboards.FileStoreManager)), cloudwatch.ProvideService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, mysql.ProvideService, mssql.ProvideService, store.ProvideEntityEventsService, dualwrite.ProvideService, httpclientprovider.New, wire.Bind(new(httpclient.Provider), new(*httpclient2.Provider)), serverlock.ProvideService, wire.Bind(new(installsync.ServerLock), new(*serverlock.ServerLockService)), annotationsimpl.ProvideCleanupService, wire.Bind(new(annotations.Cleaner), new(*annotationsimpl.CleanupServiceImpl)), cleanup.ProvideService, shorturlimpl.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturlimpl.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), correlations.ProvideService, wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, wire.Bind(new(remotecache.CacheStorage), new(*remotecache.RemoteCache)), authinfoimpl.ProvideService, wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, sort.ProvideService, search2.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, store.ProvideSystemUsersService, live.ProvideService, live.ProvideDashboardActivityChannel, pushhttp.ProvideService, contexthandler.ProvideService, service12.ProvideService, wire.Bind(new(service12.LDAP), new(*service12.LDAPImpl)), jwt.ProvideService, wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), store2.ProvideDBStore, image.ProvideDeleteExpiredService, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, notifications.ProvideSmtpService, github.ProvideFactory, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), withOTelSet, testdatasource.ProvideService, api4.ProvideService, opentsdb.ProvideService, socialimpl.ProvideService, influxdb.ProvideService, wire.Bind(new(social.Service), new(*socialimpl.SocialService)), tempo.ProvideService, loki.ProvideService, graphite.ProvideService, prometheus.ProvideService, elasticsearch.ProvideService, pyroscope.ProvideService, parca.ProvideService, zipkin.ProvideService, jaeger.ProvideService, service9.ProvideCacheService, wire.Bind(new(datasources.CacheService), new(*service9.CacheServiceImpl)), service2.ProvideEncryptionService, wire.Bind(new(encryption2.Internal), new(*service2.Service)), manager.ProvideSecretsService, wire.Bind(new(secrets.Service), new(*manager.SecretsService)), database.ProvideSecretsStore, wire.Bind(new(secrets.Store), new(*database.SecretsStoreImpl)), garbagecollectionworker.ProvideWorker, grafanads.ProvideService, wire.Bind(new(dashboardsnapshots.Store), new(*database5.DashboardSnapshotStore)), database5.ProvideStore, wire.Bind(new(dashboardsnapshots.Service), new(*service10.ServiceImpl)), service10.ProvideService, service9.ProvideDataSourceRetriever, service9.ProvideService, wire.Bind(new(datasources.DataSourceService), new(*service9.Service)), service9.ProvideLegacyDataSourceLookup, retriever.ProvideService, wire.Bind(new(serviceaccounts.ServiceAccountRetriever), new(*retriever.Service)), ossaccesscontrol.ProvideServiceAccountPermissions, wire.Bind(new(accesscontrol.ServiceAccountPermissionsService), new(*ossaccesscontrol.ServiceAccountPermissionsService)), manager3.ProvideServiceAccountsService, proxy.ProvideServiceAccountsProxy, wire.Bind(new(serviceaccounts.Service), new(*proxy.ServiceAccountsProxy)), dsquerierclient.NewNullQSDatasourceClientBuilder, expr.ProvideService, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, service7.ProvideDashboardServiceImpl, wire.Bind(new(dashboards2.PermissionsRegistrationService), new(*service7.DashboardServiceImpl)), service7.ProvideDashboardService, service7.ProvideDashboardProvisioningService, service7.ProvideDashboardPluginService, service7.ProvideDashboardAccessService, database2.ProvideDashboardStore, folderimpl.ProvideService, wire.Bind(new(folder.Service), new(*folderimpl.Service)), wire.Bind(new(folder.LegacyService), new(*folderimpl.Service)), folderimpl.ProvideStore, wire.Bind(new(folder.Store), new(*folderimpl.FolderStoreImpl)), service11.ProvideService, wire.Bind(new(dashboardimport.Service), new(*service11.ImportDashboardService)), service8.ProvideService, wire.Bind(new(plugindashboards.Service), new(*service8.Service)), service8.ProvideDashboardUpdater, kvstore2.ProvideService, avatar.ProvideAvatarCacheServer, statscollector.ProvideService, csrf.ProvideCSRFFilter, wire.Bind(new(csrf.Service), new(*csrf.CSRF)), ossaccesscontrol.ProvideTeamPermissions, wire.Bind(new(accesscontrol.TeamPermissionsService), new(*ossaccesscontrol.TeamPermissionsService)), ossaccesscontrol.ProvideFolderPermissions, wire.Bind(new(accesscontrol.FolderPermissionsService), new(*ossaccesscontrol.FolderPermissionsService)), ossaccesscontrol.ProvideDashboardPermissions, wire.Bind(new(accesscontrol.DashboardPermissionsService), new(*ossaccesscontrol.DashboardPermissionsService)), ossaccesscontrol.ProvideReceiverPermissionsService, wire.Bind(new(accesscontrol.ReceiverPermissionsService), new(*ossaccesscontrol.ReceiverPermissionsService)), starimpl.ProvideService, playlistimpl.ProvideService, apikeyimpl.ProvideService, dashverimpl.ProvideService, service3.ProvideService, wire.Bind(new(publicdashboards.Service), new(*service3.PublicDashboardServiceImpl)), database3.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*database3.PublicDashboardStoreImpl)), metric.ProvideService, api2.ProvideApi, api3.ProvideApi, userimpl.ProvideService, orgimpl.ProvideService, orgimpl.ProvideDeletionService, statsimpl.ProvideService, grpccontext.ProvideContextHandler, grpcserver.ProvideHealthService, grpcserver.ProvideReflectionService, resolver.ProvideEntityReferenceResolver, teamimpl.ProvideService, teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), migrations3.ProvideDataSourceMigrationService, migrations3.ProvideSecretMigrationProvider, wire.Bind(new(migrations3.SecretMigrationProvider), new(*migrations3.SecretMigrationProviderImpl)), promtypemigration.ProvideAzurePromMigrationService, promtypemigration.ProvideAmazonPromMigrationService, promtypemigration.ProvidePromTypeMigrationProvider, wire.Bind(new(promtypemigration.PromTypeMigrationProvider), new(*promtypemigration.PromTypeMigrationProviderImpl)), resourcepermissions.NewActionSetService, wire.Bind(new(accesscontrol.ActionResolver), new(resourcepermissions.ActionSetService)), wire.Bind(new(pluginaccesscontrol.ActionSetRegistry), new(resourcepermissions.ActionSetService)), permreg.ProvidePermissionRegistry, acimpl.ProvideAccessControl, accesscontrol.ProvideFixedRolesLoader, dualwrite2.ProvideZanzanaReconciler, navtreeimpl.ProvideService, wire.Bind(new(accesscontrol.AccessControl), new(*acimpl.AccessControl)), wire.Bind(new(notifications.TempUserStore), new(tempuser.Service)), tagimpl.ProvideService, wire.Bind(new(tag.Service), new(*tagimpl.Service)), authnimpl.ProvideService, authnimpl.ProvideIdentitySynchronizer, authnimpl.ProvideAuthnService, authnimpl.ProvideAuthnServiceAuthenticateOnly, authnimpl.ProvideRegistration, supportbundlesimpl.ProvideService, extsvcaccounts.ProvideExtSvcAccountsService, wire.Bind(new(serviceaccounts.ExtSvcAccountsService), new(*extsvcaccounts.ExtSvcAccountsService)), registry2.ProvideExtSvcRegistry, wire.Bind(new(extsvcauth.ExternalServiceRegistry), new(*registry2.Registry)), anonstore.ProvideAnonDBStore, wire.Bind(new(anonstore.AnonStore), new(*anonstore.AnonDBStore)), loggermw.Provide, slogadapter.Provide, signingkeysimpl.ProvideEmbeddedSigningKeysService, wire.Bind(new(signingkeys.Service), new(*signingkeysimpl.Service)), ssosettingsimpl.ProvideService, wire.Bind(new(ssosettings.Service), new(*ssosettingsimpl.Service)), idimpl.ProvideService, wire.Bind(new(auth.IDService), new(*idimpl.Service)), cloudmigrationimpl.ProvideService, caching.ProvideCachingServiceClient, userimpl.ProvideVerifier, connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, metadata.ProvideSecureValueMetadataStorage, metadata.ProvideKeeperMetadataStorage, metadata.ProvideDecryptStorage, decrypt.ProvideDecryptAuthorizer, wire.Value([]decrypt.ExtraOwnerDecrypter(nil)), decrypt.ProvideDecryptService, inline.ProvideInlineSecureValueService, encryption.ProvideDataKeyStorage, encryption.ProvideGlobalDataKeyStorage, encryption.ProvideEncryptedValueStorage, encryption.ProvideGlobalEncryptedValueStorage, encryption.ProvideEncryptedValueMigrationExecutor, service5.ProvideSecureValueService, validator.ProvideKeeperValidator, validator.ProvideSecureValueValidator, mutator.ProvideKeeperMutator, mutator.ProvideSecureValueMutator, migrator.NewWithEngine, database4.ProvideDatabase, clock.ProvideClock, wire.Bind(new(contracts.Database), new(*database4.Database)), wire.Bind(new(contracts.Clock), new(*clock.Clock)), manager2.ProvideEncryptionManager, service4.ProvideAESGCMCipherService, resource.ProvideStorageMetrics, resource.ProvideIndexMetrics, migrations2.ProvideUnifiedStorageMigrationService, apiserver.WireSet, apiregistry.WireSet, appregistry.WireSet, client.ProvideK8sClientWithFallback) +var wireBasicSet = wire.NewSet(annotationsimpl.ProvideService, wire.Bind(new(annotations.Repository), new(*annotationsimpl.RepositoryImpl)), New, api.ProvideHTTPServer, query.ProvideService, wire.Bind(new(query.Service), new(*query.ServiceImpl)), bus.ProvideBus, wire.Bind(new(bus.Bus), new(*bus.InProcBus)), rendering.ProvideService, wire.Bind(new(rendering.Service), new(*rendering.RenderingService)), routing.ProvideRegister, wire.Bind(new(routing.RouteRegister), new(*routing.RouteRegisterImpl)), hooks.ProvideService, kvstore.ProvideService, localcache.ProvideService, bundleregistry.ProvideService, wire.Bind(new(supportbundles.Service), new(*bundleregistry.Service)), updatemanager.ProvideGrafanaService, updatemanager.ProvidePluginsService, service.ProvideService, wire.Bind(new(usagestats.Service), new(*service.UsageStats)), validator3.ProvideService, provisioning.ProvideStubProvisioningService, legacy.ProvideMigratorDashboardAccessor, migrations2.ProvideUnifiedMigrator, pluginsintegration.WireSet, dashboards.ProvideFileStoreManager, wire.Bind(new(dashboards.FileStore), new(*dashboards.FileStoreManager)), cloudwatch.ProvideService, cloudmonitoring.ProvideService, azuremonitor.ProvideService, postgres.ProvideService, mysql.ProvideService, mssql.ProvideService, store.ProvideEntityEventsService, dualwrite.ProvideService, httpclientprovider.New, wire.Bind(new(httpclient.Provider), new(*httpclient2.Provider)), serverlock.ProvideService, wire.Bind(new(installsync.ServerLock), new(*serverlock.ServerLockService)), annotationsimpl.ProvideCleanupService, wire.Bind(new(annotations.Cleaner), new(*annotationsimpl.CleanupServiceImpl)), cleanup.ProvideService, shorturlimpl.ProvideService, wire.Bind(new(shorturls.Service), new(*shorturlimpl.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), correlations.ProvideService, wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, wire.Bind(new(remotecache.CacheStorage), new(*remotecache.RemoteCache)), authinfoimpl.ProvideService, wire.Bind(new(login.AuthInfoService), new(*authinfoimpl.Service)), authinfoimpl.ProvideStore, datasourceproxy.ProvideService, sort.ProvideService, search2.ProvideService, searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, store.ProvideSystemUsersService, live.ProvideService, live.ProvideDashboardActivityChannel, pushhttp.ProvideService, contexthandler.ProvideService, service12.ProvideService, wire.Bind(new(service12.LDAP), new(*service12.LDAPImpl)), jwt.ProvideService, wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), store2.ProvideDBStore, image.ProvideDeleteExpiredService, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), libraryelements.ProvideService, wire.Bind(new(libraryelements.Service), new(*libraryelements.LibraryElementService)), notifications.ProvideService, notifications.ProvideSmtpService, github.ProvideFactory, github2.ProvideFactory, tracing.ProvideService, tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), withOTelSet, testdatasource.ProvideService, api4.ProvideService, opentsdb.ProvideService, socialimpl.ProvideService, influxdb.ProvideService, wire.Bind(new(social.Service), new(*socialimpl.SocialService)), tempo.ProvideService, loki.ProvideService, graphite.ProvideService, prometheus.ProvideService, elasticsearch.ProvideService, pyroscope.ProvideService, parca.ProvideService, zipkin.ProvideService, jaeger.ProvideService, service9.ProvideCacheService, wire.Bind(new(datasources.CacheService), new(*service9.CacheServiceImpl)), service2.ProvideEncryptionService, wire.Bind(new(encryption2.Internal), new(*service2.Service)), manager.ProvideSecretsService, wire.Bind(new(secrets.Service), new(*manager.SecretsService)), database.ProvideSecretsStore, wire.Bind(new(secrets.Store), new(*database.SecretsStoreImpl)), garbagecollectionworker.ProvideWorker, grafanads.ProvideService, wire.Bind(new(dashboardsnapshots.Store), new(*database5.DashboardSnapshotStore)), database5.ProvideStore, wire.Bind(new(dashboardsnapshots.Service), new(*service10.ServiceImpl)), service10.ProvideService, service9.ProvideDataSourceRetriever, service9.ProvideService, wire.Bind(new(datasources.DataSourceService), new(*service9.Service)), service9.ProvideLegacyDataSourceLookup, retriever.ProvideService, wire.Bind(new(serviceaccounts.ServiceAccountRetriever), new(*retriever.Service)), ossaccesscontrol.ProvideServiceAccountPermissions, wire.Bind(new(accesscontrol.ServiceAccountPermissionsService), new(*ossaccesscontrol.ServiceAccountPermissionsService)), manager3.ProvideServiceAccountsService, proxy.ProvideServiceAccountsProxy, wire.Bind(new(serviceaccounts.Service), new(*proxy.ServiceAccountsProxy)), dsquerierclient.NewNullQSDatasourceClientBuilder, expr.ProvideService, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, service7.ProvideDashboardServiceImpl, wire.Bind(new(dashboards2.PermissionsRegistrationService), new(*service7.DashboardServiceImpl)), service7.ProvideDashboardService, service7.ProvideDashboardProvisioningService, service7.ProvideDashboardPluginService, service7.ProvideDashboardAccessService, database2.ProvideDashboardStore, folderimpl.ProvideService, wire.Bind(new(folder.Service), new(*folderimpl.Service)), wire.Bind(new(folder.LegacyService), new(*folderimpl.Service)), folderimpl.ProvideStore, wire.Bind(new(folder.Store), new(*folderimpl.FolderStoreImpl)), service11.ProvideService, wire.Bind(new(dashboardimport.Service), new(*service11.ImportDashboardService)), service8.ProvideService, wire.Bind(new(plugindashboards.Service), new(*service8.Service)), service8.ProvideDashboardUpdater, kvstore2.ProvideService, avatar.ProvideAvatarCacheServer, statscollector.ProvideService, csrf.ProvideCSRFFilter, wire.Bind(new(csrf.Service), new(*csrf.CSRF)), ossaccesscontrol.ProvideTeamPermissions, wire.Bind(new(accesscontrol.TeamPermissionsService), new(*ossaccesscontrol.TeamPermissionsService)), ossaccesscontrol.ProvideFolderPermissions, wire.Bind(new(accesscontrol.FolderPermissionsService), new(*ossaccesscontrol.FolderPermissionsService)), ossaccesscontrol.ProvideDashboardPermissions, wire.Bind(new(accesscontrol.DashboardPermissionsService), new(*ossaccesscontrol.DashboardPermissionsService)), ossaccesscontrol.ProvideReceiverPermissionsService, wire.Bind(new(accesscontrol.ReceiverPermissionsService), new(*ossaccesscontrol.ReceiverPermissionsService)), starimpl.ProvideService, playlistimpl.ProvideService, apikeyimpl.ProvideService, dashverimpl.ProvideService, service3.ProvideService, wire.Bind(new(publicdashboards.Service), new(*service3.PublicDashboardServiceImpl)), database3.ProvideStore, wire.Bind(new(publicdashboards.Store), new(*database3.PublicDashboardStoreImpl)), metric.ProvideService, api2.ProvideApi, api3.ProvideApi, userimpl.ProvideService, orgimpl.ProvideService, orgimpl.ProvideDeletionService, statsimpl.ProvideService, grpccontext.ProvideContextHandler, grpcserver.ProvideHealthService, grpcserver.ProvideReflectionService, resolver.ProvideEntityReferenceResolver, teamimpl.ProvideService, teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), migrations3.ProvideDataSourceMigrationService, migrations3.ProvideSecretMigrationProvider, wire.Bind(new(migrations3.SecretMigrationProvider), new(*migrations3.SecretMigrationProviderImpl)), promtypemigration.ProvideAzurePromMigrationService, promtypemigration.ProvideAmazonPromMigrationService, promtypemigration.ProvidePromTypeMigrationProvider, wire.Bind(new(promtypemigration.PromTypeMigrationProvider), new(*promtypemigration.PromTypeMigrationProviderImpl)), resourcepermissions.NewActionSetService, wire.Bind(new(accesscontrol.ActionResolver), new(resourcepermissions.ActionSetService)), wire.Bind(new(pluginaccesscontrol.ActionSetRegistry), new(resourcepermissions.ActionSetService)), permreg.ProvidePermissionRegistry, acimpl.ProvideAccessControl, accesscontrol.ProvideFixedRolesLoader, dualwrite2.ProvideZanzanaReconciler, navtreeimpl.ProvideService, wire.Bind(new(accesscontrol.AccessControl), new(*acimpl.AccessControl)), wire.Bind(new(notifications.TempUserStore), new(tempuser.Service)), tagimpl.ProvideService, wire.Bind(new(tag.Service), new(*tagimpl.Service)), authnimpl.ProvideService, authnimpl.ProvideIdentitySynchronizer, authnimpl.ProvideAuthnService, authnimpl.ProvideAuthnServiceAuthenticateOnly, authnimpl.ProvideRegistration, supportbundlesimpl.ProvideService, extsvcaccounts.ProvideExtSvcAccountsService, wire.Bind(new(serviceaccounts.ExtSvcAccountsService), new(*extsvcaccounts.ExtSvcAccountsService)), registry2.ProvideExtSvcRegistry, wire.Bind(new(extsvcauth.ExternalServiceRegistry), new(*registry2.Registry)), anonstore.ProvideAnonDBStore, wire.Bind(new(anonstore.AnonStore), new(*anonstore.AnonDBStore)), loggermw.Provide, slogadapter.Provide, signingkeysimpl.ProvideEmbeddedSigningKeysService, wire.Bind(new(signingkeys.Service), new(*signingkeysimpl.Service)), ssosettingsimpl.ProvideService, wire.Bind(new(ssosettings.Service), new(*ssosettingsimpl.Service)), idimpl.ProvideService, wire.Bind(new(auth.IDService), new(*idimpl.Service)), cloudmigrationimpl.ProvideService, caching.ProvideCachingServiceClient, userimpl.ProvideVerifier, connectors.ProvideOrgRoleMapper, wire.Bind(new(user.Verifier), new(*userimpl.Verifier)), authz.WireSet, metadata.ProvideSecureValueMetadataStorage, metadata.ProvideKeeperMetadataStorage, metadata.ProvideDecryptStorage, decrypt.ProvideDecryptAuthorizer, wire.Value([]decrypt.ExtraOwnerDecrypter(nil)), decrypt.ProvideDecryptService, inline.ProvideInlineSecureValueService, encryption.ProvideDataKeyStorage, encryption.ProvideGlobalDataKeyStorage, encryption.ProvideEncryptedValueStorage, encryption.ProvideGlobalEncryptedValueStorage, encryption.ProvideEncryptedValueMigrationExecutor, service5.ProvideSecureValueService, validator.ProvideKeeperValidator, validator.ProvideSecureValueValidator, mutator.ProvideKeeperMutator, mutator.ProvideSecureValueMutator, migrator.NewWithEngine, database4.ProvideDatabase, clock.ProvideClock, wire.Bind(new(contracts.Database), new(*database4.Database)), wire.Bind(new(contracts.Clock), new(*clock.Clock)), manager2.ProvideEncryptionManager, service4.ProvideAESGCMCipherService, resource.ProvideStorageMetrics, resource.ProvideIndexMetrics, migrations2.ProvideUnifiedStorageMigrationService, apiserver.WireSet, apiregistry.WireSet, appregistry.WireSet, client.ProvideK8sClientWithFallback) var wireSet = wire.NewSet( wireBasicSet, metrics.WireSet, sqlstore.ProvideService, metrics2.ProvideService, wire.Bind(new(notifications.Service), new(*notifications.NotificationService)), wire.Bind(new(notifications.WebhookSender), new(*notifications.NotificationService)), wire.Bind(new(notifications.EmailSender), new(*notifications.NotificationService)), wire.Bind(new(db.DB), new(*sqlstore.SQLStore)), prefimpl.ProvideService, oauthtoken.ProvideService, wire.Bind(new(oauthtoken.OAuthTokenService), new(*oauthtoken.Service)), wire.Bind(new(cleanup.AlertRuleService), new(*store2.DBstore)), diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index 4d70c3f2f4c..c0534f7fff4 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -72,6 +72,7 @@ import ( var provisioningExtras = wire.NewSet( extras.ProvideProvisioningOSSRepositoryExtras, + extras.ProvideProvisioningOSSConnectionExtras, ) var configProviderExtras = wire.NewSet( diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index ccbd042a0bf..15ac2e01fc5 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -879,13 +879,6 @@ var ( Owner: grafanaSharingSquad, FrontendOnly: false, }, - { - Name: "logsExploreTableDefaultVisualization", - Description: "Sets the logs table as default visualisation in logs explore", - Stage: FeatureStageExperimental, - Owner: grafanaObservabilityLogsSquad, - FrontendOnly: true, - }, { Name: "alertingListViewV2", Description: "Enables the new alert list view design", @@ -1094,13 +1087,6 @@ var ( Stage: FeatureStageExperimental, Owner: identityAccessTeam, }, - { - Name: "unifiedStorageSearch", - Description: "Enable unified storage search", - Stage: FeatureStageExperimental, - Owner: grafanaSearchAndStorageSquad, - HideFromDocs: true, - }, { Name: "unifiedStorageSearchSprinkles", Description: "Enable sprinkles on unified storage search", @@ -1169,14 +1155,6 @@ var ( Owner: identityAccessTeam, HideFromDocs: true, }, - { - Name: "exploreMetricsRelatedLogs", - Description: "Display Related Logs in Grafana Metrics Drilldown", - Stage: FeatureStageExperimental, - Owner: grafanaObservabilityMetricsSquad, - FrontendOnly: true, - HideFromDocs: false, - }, { Name: "prometheusSpecialCharsInLabelValues", Description: "Adds support for quotes and special characters in label values for Prometheus queries", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 169206faed2..12cd962dc09 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -121,7 +121,6 @@ queryLibrary,preview,@grafana/sharing-squad,false,false,false dashboardLibrary,experimental,@grafana/sharing-squad,false,false,false suggestedDashboards,experimental,@grafana/sharing-squad,false,false,false dashboardTemplates,preview,@grafana/sharing-squad,false,false,false -logsExploreTableDefaultVisualization,experimental,@grafana/observability-logs,false,false,true alertingListViewV2,privatePreview,@grafana/alerting-squad,false,false,true alertingSavedSearches,experimental,@grafana/alerting-squad,false,false,true alertingDisableSendAlertsExternal,experimental,@grafana/alerting-squad,false,false,false @@ -151,7 +150,6 @@ alertingQueryAndExpressionsStepMode,GA,@grafana/alerting-squad,false,false,true improvedExternalSessionHandling,GA,@grafana/identity-access-team,false,false,false useSessionStorageForRedirection,GA,@grafana/identity-access-team,false,false,false rolePickerDrawer,experimental,@grafana/identity-access-team,false,false,false -unifiedStorageSearch,experimental,@grafana/search-and-storage,false,false,false unifiedStorageSearchSprinkles,experimental,@grafana/search-and-storage,false,false,false managedDualWriter,experimental,@grafana/search-and-storage,false,false,false pluginsSriChecks,GA,@grafana/plugins-platform-backend,false,false,false @@ -162,7 +160,6 @@ newTimeRangeZoomShortcuts,experimental,@grafana/dataviz-squad,false,false,true azureMonitorDisableLogLimit,GA,@grafana/partner-datasources,false,false,false playlistsReconciler,experimental,@grafana/grafana-app-platform-squad,false,true,false passwordlessMagicLinkAuthentication,experimental,@grafana/identity-access-team,false,false,false -exploreMetricsRelatedLogs,experimental,@grafana/observability-metrics,false,false,true prometheusSpecialCharsInLabelValues,experimental,@grafana/oss-big-tent,false,false,true enableExtensionsAdminPage,experimental,@grafana/plugins-platform-backend,false,true,false enableSCIM,preview,@grafana/identity-access-team,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index cf0ac7b1f43..f48e74493ae 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -459,10 +459,6 @@ const ( // Enables the new role picker drawer design FlagRolePickerDrawer = "rolePickerDrawer" - // FlagUnifiedStorageSearch - // Enable unified storage search - FlagUnifiedStorageSearch = "unifiedStorageSearch" - // FlagUnifiedStorageSearchSprinkles // Enable sprinkles on unified storage search FlagUnifiedStorageSearchSprinkles = "unifiedStorageSearchSprinkles" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 7530088b237..f8b11650edc 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1421,7 +1421,8 @@ "metadata": { "name": "exploreMetricsRelatedLogs", "resourceVersion": "1764664939750", - "creationTimestamp": "2024-11-05T16:28:43Z" + "creationTimestamp": "2024-11-05T16:28:43Z", + "deletionTimestamp": "2026-01-09T22:14:53Z" }, "spec": { "description": "Display Related Logs in Grafana Metrics Drilldown", @@ -2259,7 +2260,8 @@ "metadata": { "name": "logsExploreTableDefaultVisualization", "resourceVersion": "1764664939750", - "creationTimestamp": "2024-05-02T15:28:15Z" + "creationTimestamp": "2024-05-02T15:28:15Z", + "deletionTimestamp": "2026-01-12T14:11:46Z" }, "spec": { "description": "Sets the logs table as default visualisation in logs explore", @@ -3710,7 +3712,8 @@ "metadata": { "name": "unifiedStorageSearch", "resourceVersion": "1764664939750", - "creationTimestamp": "2024-09-30T19:46:14Z" + "creationTimestamp": "2024-09-30T19:46:14Z", + "deletionTimestamp": "2026-01-12T10:02:12Z" }, "spec": { "description": "Enable unified storage search", diff --git a/pkg/services/store/kind/dashboard/ds_lookup.go b/pkg/services/store/kind/dashboard/ds_lookup.go index 5d132d569be..7330e598409 100644 --- a/pkg/services/store/kind/dashboard/ds_lookup.go +++ b/pkg/services/store/kind/dashboard/ds_lookup.go @@ -100,6 +100,9 @@ func (d *DsLookup) ByRef(ref *DataSourceRef) *DataSourceRef { if ref == nil { return d.defaultDS } + if ref.UID == "default" && ref.Type == "" { + return d.defaultDS + } key := "" if ref.UID != "" { @@ -117,7 +120,13 @@ func (d *DsLookup) ByRef(ref *DataSourceRef) *DataSourceRef { return ds } - return d.byName[key] + ds, ok = d.byName[key] + if ok { + return ds + } + + // With nothing was found (or configured), use the original reference + return ref } func (d *DsLookup) ByType(dsType string) []DataSourceRef { diff --git a/pkg/services/store/kind/dashboard/testdata/panel-with-library-panel-field-info.json b/pkg/services/store/kind/dashboard/testdata/panel-with-library-panel-field-info.json index 1ffaecb605b..2a687ad8865 100644 --- a/pkg/services/store/kind/dashboard/testdata/panel-with-library-panel-field-info.json +++ b/pkg/services/store/kind/dashboard/testdata/panel-with-library-panel-field-info.json @@ -4,8 +4,8 @@ "tags": null, "datasource": [ { - "uid": "default.uid", - "type": "default.type" + "uid": "000000001", + "type": "graphite" } ], "panels": [ @@ -16,8 +16,8 @@ "libraryPanel": "dfkljg98345dkf", "datasource": [ { - "uid": "default.uid", - "type": "default.type" + "uid": "000000001", + "type": "graphite" } ] } diff --git a/pkg/services/store/kind/dashboard/types.go b/pkg/services/store/kind/dashboard/types.go index 51aa00a79fd..c1dea30abb7 100644 --- a/pkg/services/store/kind/dashboard/types.go +++ b/pkg/services/store/kind/dashboard/types.go @@ -1,5 +1,7 @@ package dashboard +import "iter" + type PanelSummaryInfo struct { ID int64 `json:"id"` Title string `json:"title"` @@ -30,3 +32,20 @@ type DashboardSummaryInfo struct { Refresh string `json:"refresh,omitempty"` ReadOnly bool `json:"readOnly,omitempty"` // editable = false } + +func (d *DashboardSummaryInfo) PanelIterator() iter.Seq[PanelSummaryInfo] { + return func(yield func(PanelSummaryInfo) bool) { + for _, p := range d.Panels { + if len(p.Collapsed) > 0 { + for _, c := range p.Collapsed { + if !yield(c) { // NOTE, rows can only be one level deep! + return + } + } + } + if !yield(p) { + return + } + } + } +} diff --git a/pkg/storage/unified/README.md b/pkg/storage/unified/README.md index aef27df8e05..9cd0d1fd01d 100644 --- a/pkg/storage/unified/README.md +++ b/pkg/storage/unified/README.md @@ -236,7 +236,6 @@ kubernetesDashboards = true kubernetesFolders = true unifiedStorage = true unifiedStorageHistoryPruner = true -unifiedStorageSearch = true unifiedStorageSearchPermissionFiltering = false unifiedStorageSearchSprinkles = false diff --git a/pkg/storage/unified/resource/notifier.go b/pkg/storage/unified/resource/notifier.go index 5dd6a17ad29..3d3b2024d7e 100644 --- a/pkg/storage/unified/resource/notifier.go +++ b/pkg/storage/unified/resource/notifier.go @@ -78,13 +78,13 @@ func (n *notifier) Watch(ctx context.Context, opts watchOptions) <-chan Event { cache := gocache.New(cacheTTL, cacheCleanupInterval) events := make(chan Event, opts.BufferSize) - initialRV, err := n.lastEventResourceVersion(ctx) + lastRV, err := n.lastEventResourceVersion(ctx) if errors.Is(err, ErrNotFound) { - initialRV = snowflakeFromTime(time.Now()) // No events yet, start from the beginning + lastRV = 0 // No events yet, start from the beginning } else if err != nil { n.log.Error("Failed to get last event resource version", "error", err) } - lastRV := initialRV + 1 // We want to start watching from the next event + lastRV = lastRV + 1 // We want to start watching from the next event go func() { defer close(events) @@ -110,7 +110,7 @@ func (n *notifier) Watch(ctx context.Context, opts watchOptions) <-chan Event { } // Skip old events lower than the requested resource version - if evt.ResourceVersion <= initialRV { + if evt.ResourceVersion < lastRV { continue } diff --git a/pkg/storage/unified/resource/notifier_test.go b/pkg/storage/unified/resource/notifier_test.go index 060f8eecfbe..f78629ebeb7 100644 --- a/pkg/storage/unified/resource/notifier_test.go +++ b/pkg/storage/unified/resource/notifier_test.go @@ -25,7 +25,6 @@ func setupTestNotifier(t *testing.T) (*notifier, *eventStore) { return notifier, eventStore } -// nolint:unused func setupTestNotifierSqlKv(t *testing.T) (*notifier, *eventStore) { dbstore := db.InitTestDB(t) eDB, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), nil) @@ -60,8 +59,7 @@ func runNotifierTestWith(t *testing.T, storeName string, newStoreFn func(*testin func TestNotifier_lastEventResourceVersion(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierLastEventResourceVersion) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierLastEventResourceVersion) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierLastEventResourceVersion) } func testNotifierLastEventResourceVersion(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { @@ -112,8 +110,7 @@ func testNotifierLastEventResourceVersion(t *testing.T, ctx context.Context, not func TestNotifier_cachekey(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierCachekey) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierCachekey) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierCachekey) } func testNotifierCachekey(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { @@ -167,8 +164,7 @@ func testNotifierCachekey(t *testing.T, ctx context.Context, notifier *notifier, func TestNotifier_Watch_NoEvents(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierWatchNoEvents) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchNoEvents) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchNoEvents) } func testNotifierWatchNoEvents(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { @@ -209,8 +205,7 @@ func testNotifierWatchNoEvents(t *testing.T, ctx context.Context, notifier *noti func TestNotifier_Watch_WithExistingEvents(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierWatchWithExistingEvents) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchWithExistingEvents) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchWithExistingEvents) } func testNotifierWatchWithExistingEvents(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { @@ -284,8 +279,7 @@ func testNotifierWatchWithExistingEvents(t *testing.T, ctx context.Context, noti func TestNotifier_Watch_EventDeduplication(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierWatchEventDeduplication) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchEventDeduplication) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchEventDeduplication) } func testNotifierWatchEventDeduplication(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { @@ -351,8 +345,7 @@ func testNotifierWatchEventDeduplication(t *testing.T, ctx context.Context, noti func TestNotifier_Watch_ContextCancellation(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierWatchContextCancellation) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchContextCancellation) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchContextCancellation) } func testNotifierWatchContextCancellation(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { @@ -398,8 +391,7 @@ func testNotifierWatchContextCancellation(t *testing.T, ctx context.Context, not func TestNotifier_Watch_MultipleEvents(t *testing.T) { runNotifierTestWith(t, "badger", setupTestNotifier, testNotifierWatchMultipleEvents) - // enable this when sqlkv is ready - // runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchMultipleEvents) + runNotifierTestWith(t, "sqlkv", setupTestNotifierSqlKv, testNotifierWatchMultipleEvents) } func testNotifierWatchMultipleEvents(t *testing.T, ctx context.Context, notifier *notifier, eventStore *eventStore) { diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go index bca5fb98491..ade86760502 100644 --- a/pkg/storage/unified/resource/search.go +++ b/pkg/storage/unified/resource/search.go @@ -863,7 +863,7 @@ func newRebuildRequest(key NamespacedResource, minBuildTime, lastImportTime time func (s *searchSupport) getOrCreateIndex(ctx context.Context, stats *SearchStats, key NamespacedResource, reason string) (ResourceIndex, error) { if s == nil || s.search == nil { - return nil, fmt.Errorf("search is not configured properly (missing unifiedStorageSearch feature toggle?)") + return nil, fmt.Errorf("search is not configured properly (missing enable_search config?)") } ctx, span := tracer.Start(ctx, "resource.searchSupport.getOrCreateIndex") diff --git a/pkg/storage/unified/resource/storage_backend.go b/pkg/storage/unified/resource/storage_backend.go index 55843905c72..4db6da89d9a 100644 --- a/pkg/storage/unified/resource/storage_backend.go +++ b/pkg/storage/unified/resource/storage_backend.go @@ -346,7 +346,8 @@ func (k *kvStorageBackend) WriteEvent(ctx context.Context, event WriteEvent) (in return 0, fmt.Errorf("failed to write data: %w", err) } - dataKey.ResourceVersion = rvmanager.SnowflakeFromRv(rv) + rv = rvmanager.SnowflakeFromRv(rv) + dataKey.ResourceVersion = rv } else { err := k.dataStore.Save(ctx, dataKey, bytes.NewReader(event.Value)) if err != nil { diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index 254c1080653..eec7290633b 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -1253,21 +1253,23 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R queryExact.SetField(resource.SEARCH_FIELD_TITLE) queryExact.Analyzer = keyword.Name // don't analyze the query input - treat it as a single token queryExact.Operator = query.MatchQueryOperatorAnd // This doesn't make a difference for keyword analyzer, we add it just to be explicit. + searchQuery := bleve.NewDisjunctionQuery(queryExact) // Query 2: Phrase query with standard analyzer queryPhrase := bleve.NewMatchPhraseQuery(req.Query) queryPhrase.SetBoost(5.0) queryPhrase.SetField(resource.SEARCH_FIELD_TITLE) queryPhrase.Analyzer = standard.Name + searchQuery.AddQuery(queryPhrase) // Query 3: Match query with standard analyzer queryAnalyzed := bleve.NewMatchQuery(removeSmallTerms(req.Query)) queryAnalyzed.SetField(resource.SEARCH_FIELD_TITLE) + queryAnalyzed.SetBoost(2.0) queryAnalyzed.Analyzer = standard.Name queryAnalyzed.Operator = query.MatchQueryOperatorAnd // Make sure all terms from the query are matched + searchQuery.AddQuery(queryAnalyzed) - // At least one of the queries must match - searchQuery := bleve.NewDisjunctionQuery(queryExact, queryAnalyzed, queryPhrase) queries = append(queries, searchQuery) } diff --git a/pkg/storage/unified/search/bleve_test.go b/pkg/storage/unified/search/bleve_test.go index d80e35f90e5..c9c3967cd58 100644 --- a/pkg/storage/unified/search/bleve_test.go +++ b/pkg/storage/unified/search/bleve_test.go @@ -23,7 +23,6 @@ import ( "go.uber.org/goleak" authlib "github.com/grafana/authlib/types" - "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/infra/log" diff --git a/pkg/storage/unified/search/builders/dashboard.go b/pkg/storage/unified/search/builders/dashboard.go index 4f8d55111a9..a2963d71186 100644 --- a/pkg/storage/unified/search/builders/dashboard.go +++ b/pkg/storage/unified/search/builders/dashboard.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "slices" "sort" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -18,6 +19,7 @@ import ( const DASHBOARD_SCHEMA_VERSION = "schema_version" const DASHBOARD_LINK_COUNT = "link_count" const DASHBOARD_PANEL_TYPES = "panel_types" +const DASHBOARD_PANEL_TITLE = "panel_title" const DASHBOARD_DS_TYPES = "ds_types" const DASHBOARD_TRANSFORMATIONS = "transformation" const DASHBOARD_LIBRARY_PANEL_REFERENCE = "reference.LibraryPanel" @@ -53,11 +55,21 @@ func DashboardBuilder(namespaced resource.NamespacedDocumentSupplier) (resource. Type: resourcepb.ResourceTableColumnDefinition_INT32, Description: "How many links appear on the page", }, + { + Name: DASHBOARD_PANEL_TITLE, + Type: resourcepb.ResourceTableColumnDefinition_STRING, + IsArray: true, + Description: "The panel title text", + Properties: &resourcepb.ResourceTableColumnDefinition_Properties{ + Filterable: false, // full text + FreeText: true, + }, + }, { Name: DASHBOARD_PANEL_TYPES, Type: resourcepb.ResourceTableColumnDefinition_STRING, IsArray: true, - Description: "How many links appear on the page", + Description: "The panel types used in this dashboard", Properties: &resourcepb.ResourceTableColumnDefinition_Properties{ Filterable: true, }, @@ -269,14 +281,22 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou doc.Description = summary.Description doc.Tags = summary.Tags + panelTitles := []string{} panelTypes := []string{} transformations := []string{} dsTypes := []string{} - for _, p := range summary.Panels { - if p.Type != "" { + for p := range summary.PanelIterator() { + switch p.Type { + case "": // ignore + case "row": // row should map to a layout type when we support v2 constructs + default: panelTypes = append(panelTypes, p.Type) } + + if len(p.Title) > 0 { + panelTitles = append(panelTitles, p.Title) + } if len(p.Transformer) > 0 { transformations = append(transformations, p.Transformer...) } @@ -309,17 +329,20 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou resource.SEARCH_FIELD_LEGACY_ID: summary.ID, } + if len(panelTitles) > 0 { + doc.Fields[DASHBOARD_PANEL_TITLE] = panelTitles + } if len(panelTypes) > 0 { sort.Strings(panelTypes) - doc.Fields[DASHBOARD_PANEL_TYPES] = panelTypes + doc.Fields[DASHBOARD_PANEL_TYPES] = slices.Compact(panelTypes) // distinct values } if len(dsTypes) > 0 { sort.Strings(dsTypes) - doc.Fields[DASHBOARD_DS_TYPES] = dsTypes + doc.Fields[DASHBOARD_DS_TYPES] = slices.Compact(dsTypes) // distinct values } if len(transformations) > 0 { sort.Strings(transformations) - doc.Fields[DASHBOARD_TRANSFORMATIONS] = transformations + doc.Fields[DASHBOARD_TRANSFORMATIONS] = slices.Compact(transformations) // distinct values } for k, v := range s.Stats[summary.UID] { diff --git a/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa-out.json b/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa-out.json index a77725e2cc2..fdb77b02c3b 100644 --- a/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa-out.json +++ b/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa-out.json @@ -32,10 +32,16 @@ "errors_last_7_days": 1, "grafana.app/deprecatedInternalID": 141, "link_count": 0, + "panel_title": [ + "green pie", + "red pie", + "blue pie", + "collapsed row" + ], "panel_types": [ "barchart", "graph", - "row" + "pie" ], "schema_version": 38 }, @@ -46,6 +52,12 @@ "kind": "DataSource", "name": "DSUID" }, + { + "relation": "depends-on", + "group": "dashboards.grafana.app", + "kind": "LibraryPanel", + "name": "l3d2s634-fdgf-75u4-3fg3-67j966ii7jur" + }, { "relation": "depends-on", "group": "dashboards.grafana.app", diff --git a/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa.json b/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa.json index d9ecbfc6aec..24360ee929b 100644 --- a/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa.json +++ b/pkg/storage/unified/search/builders/testdata/doc/dashboard-aaa.json @@ -67,7 +67,7 @@ "name": "red pie", "uid": "e1d5f519-dabd-47c6-9ad7-83d181ce1cee" }, - "title": "green pie" + "title": "red pie" }, { "id": 7, @@ -78,6 +78,14 @@ "id": 8, "type": "graph" }, + { + "id": 20, + "type": "graph" + }, + { + "id": 30, + "type": "graph" + }, { "collapsed": true, "gridPos": { @@ -101,6 +109,10 @@ "uid": "l3d2s634-fdgf-75u4-3fg3-67j966ii7jur" }, "title": "blue pie" + }, + { + "id": 40, + "type": "pie" } ], "title": "collapsed row", diff --git a/pkg/storage/unified/search/options.go b/pkg/storage/unified/search/options.go index 20a0874b598..d450e9ae24b 100644 --- a/pkg/storage/unified/search/options.go +++ b/pkg/storage/unified/search/options.go @@ -19,7 +19,7 @@ func NewSearchOptions( ownsIndexFn func(key resource.NamespacedResource) (bool, error), ) (resource.SearchOptions, error) { //nolint:staticcheck // not yet migrated to OpenFeature - if cfg.EnableSearch || features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageSearch) || features.IsEnabledGlobally(featuremgmt.FlagProvisioning) { + if cfg.EnableSearch || features.IsEnabledGlobally(featuremgmt.FlagProvisioning) { root := cfg.IndexPath if root == "" { root = filepath.Join(cfg.DataPath, "unified-search", "bleve") diff --git a/pkg/storage/unified/search/testdata/manual-dashboard.json b/pkg/storage/unified/search/testdata/manual-dashboard.json index 4208a58dcd3..2ae346d072c 100644 --- a/pkg/storage/unified/search/testdata/manual-dashboard.json +++ b/pkg/storage/unified/search/testdata/manual-dashboard.json @@ -71,11 +71,18 @@ "description": "How many links appear on the page", "priority": 0 }, + { + "name": "panel_title", + "type": "string", + "format": "", + "description": "The panel title text", + "priority": 0 + }, { "name": "panel_types", "type": "string", "format": "", - "description": "How many links appear on the page", + "description": "The panel types used in this dashboard", "priority": 0 }, { @@ -214,6 +221,7 @@ null, null, null, + null, null ], "object": { @@ -239,6 +247,7 @@ "repo", null, null, + null, [ "timeseries" ], @@ -282,6 +291,7 @@ "repo", null, null, + null, [ "timeseries", "table" diff --git a/pkg/storage/unified/testing/storage_backend_sql_compatibility.go b/pkg/storage/unified/testing/storage_backend_sql_compatibility.go index 6584992f3cd..9066a39221c 100644 --- a/pkg/storage/unified/testing/storage_backend_sql_compatibility.go +++ b/pkg/storage/unified/testing/storage_backend_sql_compatibility.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/bwmarrin/snowflake" "github.com/stretchr/testify/require" claims "github.com/grafana/authlib/types" @@ -187,13 +186,30 @@ func runKeyPathTest(t *testing.T, backend resource.StorageBackend, nsPrefix stri // verifyKeyPath is a helper function to verify key_path generation func verifyKeyPath(t *testing.T, db sqldb.DB, ctx context.Context, key *resourcepb.ResourceKey, action string, resourceVersion int64, expectedFolder string) { + // For SQL backend (namespace contains "-sql"), resourceVersion is in microsecond format + // but key_path stores snowflake RV, so convert to snowflake + // For KV backend (namespace contains "-kv"), resourceVersion is already in snowflake format + isSqlBackend := strings.Contains(key.Namespace, "-sql") + + var keyPathRV int64 + if isSqlBackend { + // Convert microsecond RV to snowflake for key_path construction + keyPathRV = rvmanager.SnowflakeFromRv(resourceVersion) + } else { + // KV backend already provides snowflake RV + keyPathRV = resourceVersion + } + + // Build the expected key_path using DataKey format: unified/data/group/resource/namespace/name/resourceVersion~action~folder + expectedKeyPath := fmt.Sprintf("unified/data/%s/%s/%s/%s/%d~%s~%s", key.Group, key.Resource, key.Namespace, key.Name, keyPathRV, action, expectedFolder) + var query string if db.DriverName() == "postgres" { - query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = $1 AND name = $2 AND resource_version = $3" + query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE key_path = $1" } else { - query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE namespace = ? AND name = ? AND resource_version = ?" + query = "SELECT key_path, resource_version, action, folder FROM resource_history WHERE key_path = ?" } - rows, err := db.QueryContext(ctx, query, key.Namespace, key.Name, resourceVersion) + rows, err := db.QueryContext(ctx, query, expectedKeyPath) require.NoError(t, err) require.True(t, rows.Next(), "Resource not found in resource_history table - both SQL and KV backends should write to this table") @@ -220,10 +236,6 @@ func verifyKeyPath(t *testing.T, db sqldb.DB, ctx context.Context, key *resource // Verify action suffix require.Contains(t, keyPath, fmt.Sprintf("~%s~", action)) - // Verify snowflake calculation - expectedSnowflake := (((resourceVersion / 1000) - snowflake.Epoch) << (snowflake.NodeBits + snowflake.StepBits)) + (resourceVersion % 1000) - require.Contains(t, keyPath, fmt.Sprintf("/%d~", expectedSnowflake), "actual RV: %d", actualRV) - // Verify folder if specified if expectedFolder != "" { require.Equal(t, expectedFolder, actualFolder) @@ -492,10 +504,10 @@ func verifyResourceHistoryRecord(t *testing.T, record ResourceHistoryRecord, exp } // Validate previous_resource_version - // For KV backend operations, resource versions are stored as snowflake format - // but expectedPrevRV is in microsecond format, so we need to use IsRvEqual for comparison + // For KV backend operations, expectedPrevRV is now in snowflake format (returned by KV backend) + // but resource_history table stores microsecond RV, so we need to use IsRvEqual for comparison if strings.Contains(record.Namespace, "-kv") { - require.True(t, rvmanager.IsRvEqual(record.PreviousResourceVersion, expectedPrevRV), + require.True(t, rvmanager.IsRvEqual(expectedPrevRV, record.PreviousResourceVersion), "Previous resource version should match (KV backend snowflake format)") } else { require.Equal(t, expectedPrevRV, record.PreviousResourceVersion) @@ -505,9 +517,10 @@ func verifyResourceHistoryRecord(t *testing.T, record ResourceHistoryRecord, exp require.Equal(t, expectedGeneration, record.Generation) // Validate resource_version - // For KV backend operations, resource versions are stored as snowflake format + // For KV backend operations, expectedRV is now in snowflake format (returned by KV backend) + // but resource_history table stores microsecond RV, so we need to use IsRvEqual for comparison if strings.Contains(record.Namespace, "-kv") { - require.True(t, rvmanager.IsRvEqual(record.ResourceVersion, expectedRV), + require.True(t, rvmanager.IsRvEqual(expectedRV, record.ResourceVersion), "Resource version should match (KV backend snowflake format)") } else { require.Equal(t, expectedRV, record.ResourceVersion) @@ -574,7 +587,7 @@ func verifyResourceTable(t *testing.T, db sqldb.DB, namespace string, resources // Resource version should match the expected version for test-resource-3 (updated version) expectedRV := resourceVersions[2][1] // test-resource-3's update version if strings.Contains(namespace, "-kv") { - require.True(t, rvmanager.IsRvEqual(record.ResourceVersion, expectedRV), + require.True(t, rvmanager.IsRvEqual(expectedRV, record.ResourceVersion), "Resource version should match (KV backend snowflake format)") } else { require.Equal(t, expectedRV, record.ResourceVersion) @@ -625,9 +638,16 @@ func verifyResourceVersionTable(t *testing.T, db sqldb.DB, namespace string, res // The resource_version table should contain the latest RV for the group+resource // It might be slightly higher due to RV manager operations, so check it's at least our max - require.GreaterOrEqual(t, record.ResourceVersion, maxRV, "resource_version should be at least the latest RV we tracked") - // But it shouldn't be too much higher (within a reasonable range) - require.LessOrEqual(t, record.ResourceVersion, maxRV+100, "resource_version shouldn't be much higher than expected") + // For KV backend, maxRV is in snowflake format but record.ResourceVersion is in microsecond format + // Use IsRvEqual for proper comparison between different RV formats + isKvBackend := strings.Contains(namespace, "-kv") + recordResourceVersion := record.ResourceVersion + if isKvBackend { + recordResourceVersion = rvmanager.SnowflakeFromRv(record.ResourceVersion) + } + + require.Less(t, recordResourceVersion, int64(9223372036854775807), "resource_version should be reasonable") + require.Greater(t, recordResourceVersion, maxRV, "resource_version should be at least the latest RV we tracked") } // runTestCrossBackendConsistency tests basic consistency between SQL and KV backends (lightweight) diff --git a/pkg/storage/unified/testing/storage_backend_test.go b/pkg/storage/unified/testing/storage_backend_test.go index 092cd476b52..3046967adee 100644 --- a/pkg/storage/unified/testing/storage_backend_test.go +++ b/pkg/storage/unified/testing/storage_backend_test.go @@ -38,7 +38,6 @@ func TestBadgerKVStorageBackend(t *testing.T) { func TestSQLKVStorageBackend(t *testing.T) { skipTests := map[string]bool{ - TestHappyPath: true, TestWatchWriteEvents: true, TestList: true, TestBlobSupport: true, @@ -51,21 +50,24 @@ func TestSQLKVStorageBackend(t *testing.T) { TestGetResourceLastImportTime: true, TestOptimisticLocking: true, } - // without RvManager - RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend { - backend, _ := NewTestSqlKvBackend(t, ctx, false) - return backend - }, &TestOptions{ - NSPrefix: "sqlkvstorage-test", - SkipTests: skipTests, + + t.Run("Without RvManager", func(t *testing.T) { + RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend { + backend, _ := NewTestSqlKvBackend(t, ctx, false) + return backend + }, &TestOptions{ + NSPrefix: "sqlkvstorage-test", + SkipTests: skipTests, + }) }) - // with RvManager - RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend { - backend, _ := NewTestSqlKvBackend(t, ctx, true) - return backend - }, &TestOptions{ - NSPrefix: "sqlkvstorage-withrvmanager-test", - SkipTests: skipTests, + t.Run("With RvManager", func(t *testing.T) { + RunStorageBackendTest(t, func(ctx context.Context) resource.StorageBackend { + backend, _ := NewTestSqlKvBackend(t, ctx, true) + return backend + }, &TestOptions{ + NSPrefix: "sqlkvstorage-withrvmanager-test", + SkipTests: skipTests, + }) }) } diff --git a/pkg/tests/apis/dashboard/search_test.go b/pkg/tests/apis/dashboard/search_test.go index 3fe35c51b3a..2227c67287e 100644 --- a/pkg/tests/apis/dashboard/search_test.go +++ b/pkg/tests/apis/dashboard/search_test.go @@ -4,10 +4,15 @@ import ( "context" "encoding/json" "fmt" + "io/fs" + "math" "net/http" + "os" + "path/filepath" "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" @@ -16,12 +21,167 @@ import ( dashboardV0 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apiserver/rest" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tests/apis" "github.com/grafana/grafana/pkg/tests/testinfra" "github.com/grafana/grafana/pkg/util/testutil" ) +func TestIntegrationSearchDevDashboards(t *testing.T) { + testutil.SkipIntegrationTestInShortMode(t) + ctx := context.Background() + + helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + DisableDataMigrations: true, + AppModeProduction: true, + DisableAnonymous: true, + APIServerStorageType: "unified", + UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{ + "dashboards.dashboard.grafana.app": {DualWriterMode: rest.Mode5}, + "folders.folder.grafana.app": {DualWriterMode: rest.Mode5}, + }, + UnifiedStorageEnableSearch: true, + }) + defer helper.Shutdown() + + // Create devenv dashboards from legacy API + cfg := dynamic.ConfigFor(helper.Org1.Admin.NewRestConfig()) + cfg.GroupVersion = &dashboardV0.GroupVersion + adminClient, err := k8srest.RESTClientFor(cfg) + require.NoError(t, err) + adminClient.Get() + + fileCount := 0 + devenv := "../../../../devenv/dev-dashboards/panel-timeseries" + err = filepath.WalkDir(devenv, func(p string, d fs.DirEntry, e error) error { + require.NoError(t, err) + if d.IsDir() || filepath.Ext(d.Name()) != ".json" { + return nil + } + + // use the filename as UID + uid := strings.TrimSuffix(d.Name(), ".json") + if len(uid) > 40 { + uid = uid[:40] // avoid uid too long, max 40 characters + } + + // nolint:gosec + data, err := os.ReadFile(p) + require.NoError(t, err) + + cmd := dashboards.SaveDashboardCommand{ + Dashboard: &simplejson.Json{}, + Overwrite: true, + } + err = cmd.Dashboard.FromDB(data) + require.NoError(t, err) + cmd.Dashboard.Set("id", nil) + cmd.Dashboard.Set("uid", uid) + data, err = json.Marshal(cmd) + require.NoError(t, err) + + var statusCode int + result := adminClient.Post().AbsPath("api", "dashboards", "db"). + Body(data). + SetHeader("Content-type", "application/json"). + Do(ctx). + StatusCode(&statusCode) + require.NoError(t, result.Error(), "file: [%d] %s [status:%d]", fileCount, d.Name(), statusCode) + require.Equal(t, int(http.StatusOK), statusCode) + fileCount++ + return nil + }) + require.NoError(t, err) + require.Equal(t, 16, fileCount, "file count from %s", devenv) + + // Helper to call search + callSearch := func(user apis.User, params string) dashboardV0.SearchResults { + require.NotNil(t, user) + ns := user.Identity.GetNamespace() + cfg := dynamic.ConfigFor(user.NewRestConfig()) + cfg.GroupVersion = &dashboardV0.GroupVersion + restClient, err := k8srest.RESTClientFor(cfg) + require.NoError(t, err) + + var statusCode int + req := restClient.Get().AbsPath("apis", "dashboard.grafana.app", "v0alpha1", "namespaces", ns, "search"). + Param("limit", "1000"). + Param("type", "dashboard") // Only search dashboards + + for kv := range strings.SplitSeq(params, "&") { + if kv == "" { + continue + } + parts := strings.SplitN(kv, "=", 2) + if len(parts) == 2 { + req = req.Param(parts[0], parts[1]) + } + } + res := req.Do(ctx).StatusCode(&statusCode) + require.NoError(t, res.Error()) + require.Equal(t, int(http.StatusOK), statusCode) + var sr dashboardV0.SearchResults + raw, err := res.Raw() + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &sr)) + + // Normalize scores and query cost for snapshot comparison + sr.QueryCost = 0 // this depends on the hardware + sr.MaxScore = roundTo(sr.MaxScore, 3) + for i := range sr.Hits { + sr.Hits[i].Score = roundTo(sr.Hits[i].Score, 3) // 0.6250571494814442 -> 0.625 + } + return sr + } + + // Compare a results to snapshots + testCases := []struct { + name string + user apis.User + params string + }{ + { + name: "all", + user: helper.Org1.Admin, + params: "", // only dashboards + }, + { + name: "simple-query", + user: helper.Org1.Admin, + params: "query=stacking", + }, + { + name: "with-text-panel", + user: helper.Org1.Admin, + params: "field=panel_types&panelType=text", + }, + } + for i, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + res := callSearch(tc.user, tc.params) + jj, err := json.MarshalIndent(res, "", " ") + require.NoError(t, err) + + fname := fmt.Sprintf("testdata/searchV0/t%02d-%s.json", i, tc.name) + // nolint:gosec + snapshot, err := os.ReadFile(fname) + if err != nil { + assert.Failf(t, "Failed to read snapshot", "file: %s", fname) + err = os.WriteFile(fname, jj, 0o644) + require.NoErrorf(t, err, "Failed to write snapshot file %s", fname) + return + } + + if !assert.JSONEq(t, string(snapshot), string(jj)) { + err = os.WriteFile(fname, jj, 0o644) + require.NoErrorf(t, err, "Failed to write snapshot file %s", fname) + } + }) + } +} + func TestIntegrationSearchPermissionFiltering(t *testing.T) { testutil.SkipIntegrationTestInShortMode(t) @@ -285,3 +445,11 @@ func setFolderPermissions(t *testing.T, helper *apis.K8sTestHelper, actingUser a require.Equal(t, http.StatusOK, resp.Response.StatusCode, "Failed to set permissions for folder %s", folderUID) } + +// roundTo rounds a float64 to a specified number of decimal places. +func roundTo(n float64, decimals uint32) float64 { + // Calculate the power of 10 for the desired number of decimals + scale := math.Pow(10, float64(decimals)) + // Multiply, round to the nearest integer, and then divide back + return math.Round(n*scale) / scale +} diff --git a/pkg/tests/apis/dashboard/testdata/searchV0/t00-all.json b/pkg/tests/apis/dashboard/testdata/searchV0/t00-all.json new file mode 100644 index 00000000000..35b7cff0302 --- /dev/null +++ b/pkg/tests/apis/dashboard/testdata/searchV0/t00-all.json @@ -0,0 +1,165 @@ +{ + "totalHits": 16, + "hits": [ + { + "resource": "dashboards", + "name": "timeseries", + "title": "Panel Tests - Graph NG", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-by-value-color-schemes", + "title": "Panel Tests - Graph NG - By value color schemes", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-nulls", + "title": "Panel Tests - Graph NG - Discrete panels", + "tags": [ + "gdev", + "panel-tests", + "graph-ng", + "timeseries", + "trend", + "state-timeline", + "transform" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-gradient-area", + "title": "Panel Tests - Graph NG - Gradient Area Fills", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-soft-limits", + "title": "Panel Tests - Graph NG - softMin/softMax", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-yaxis-ticks", + "title": "Panel Tests - Graph NG - Y axis ticks", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-hue-gradients", + "title": "Panel Tests - GraphNG - Hue Gradients", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-time", + "title": "Panel Tests - GraphNG - Time Axis", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-thresholds", + "title": "Panel Tests - GraphNG Thresholds", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-shared-tooltip-cursor-positio", + "title": "Panel Tests - shared tooltips cursor positioning", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-bars-high-density", + "title": "Panel Tests - TimeSeries - bars high density (stroke + fill)", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-out-of-rage", + "title": "Panel Tests - Timeseries - Out of range", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-stacking", + "title": "Panel Tests - TimeSeries - stacking", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-formats", + "title": "Panel Tests - Timeseries - Supported input formats" + }, + { + "resource": "dashboards", + "name": "timeseries-stacking2", + "title": "TimeSeries \u0026 BarChart Stacking", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + }, + { + "resource": "dashboards", + "name": "timeseries-y-ticks-zero-decimals", + "title": "Zero Decimals Y Ticks", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ] + } + ], + "maxScore": 1 +} \ No newline at end of file diff --git a/pkg/tests/apis/dashboard/testdata/searchV0/t01-simple-query.json b/pkg/tests/apis/dashboard/testdata/searchV0/t01-simple-query.json new file mode 100644 index 00000000000..6c9a935dfe8 --- /dev/null +++ b/pkg/tests/apis/dashboard/testdata/searchV0/t01-simple-query.json @@ -0,0 +1,28 @@ +{ + "totalHits": 2, + "hits": [ + { + "resource": "dashboards", + "name": "timeseries-stacking", + "title": "Panel Tests - TimeSeries - stacking", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ], + "score": 0.658 + }, + { + "resource": "dashboards", + "name": "timeseries-stacking2", + "title": "TimeSeries \u0026 BarChart Stacking", + "tags": [ + "gdev", + "panel-tests", + "graph-ng" + ], + "score": 0.625 + } + ], + "maxScore": 0.658 +} \ No newline at end of file diff --git a/pkg/tests/apis/dashboard/testdata/searchV0/t02-with-text-panel.json b/pkg/tests/apis/dashboard/testdata/searchV0/t02-with-text-panel.json new file mode 100644 index 00000000000..b38cfb14b40 --- /dev/null +++ b/pkg/tests/apis/dashboard/testdata/searchV0/t02-with-text-panel.json @@ -0,0 +1,18 @@ +{ + "totalHits": 1, + "hits": [ + { + "resource": "dashboards", + "name": "timeseries-formats", + "title": "Panel Tests - Timeseries - Supported input formats", + "field": { + "panel_types": [ + "table", + "text", + "timeseries" + ] + } + } + ], + "maxScore": 1.778 +} \ No newline at end of file diff --git a/pkg/tests/apis/datasource/testdata_test.go b/pkg/tests/apis/datasource/testdata_test.go index 9a94bea5dce..85450ec6536 100644 --- a/pkg/tests/apis/datasource/testdata_test.go +++ b/pkg/tests/apis/datasource/testdata_test.go @@ -62,7 +62,7 @@ func TestIntegrationTestDatasource(t *testing.T) { t.Run("Admin configs", func(t *testing.T) { client := helper.Org1.Admin.ResourceClient(t, schema.GroupVersionResource{ - Group: "testdata.datasource.grafana.app", + Group: "grafana-testdata-datasource.datasource.grafana.app", Version: "v0alpha1", Resource: "datasources", }).Namespace("default") @@ -92,7 +92,7 @@ func TestIntegrationTestDatasource(t *testing.T) { t.Run("Call subresources", func(t *testing.T) { client := helper.Org1.Admin.ResourceClient(t, schema.GroupVersionResource{ - Group: "testdata.datasource.grafana.app", + Group: "grafana-testdata-datasource.datasource.grafana.app", Version: "v0alpha1", Resource: "datasources", }).Namespace("default") @@ -128,7 +128,7 @@ func TestIntegrationTestDatasource(t *testing.T) { raw := apis.DoRequest[any](helper, apis.RequestParams{ User: helper.Org1.Admin, Method: "GET", - Path: "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/default/datasources/test/resource", + Path: "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/default/datasources/test/resource", }, nil) // endpoint is disabled currently because it has not been // sufficiently tested. diff --git a/pkg/tests/apis/folder/folders_test.go b/pkg/tests/apis/folder/folders_test.go index ce7dd5f6795..cdcc798c93a 100644 --- a/pkg/tests/apis/folder/folders_test.go +++ b/pkg/tests/apis/folder/folders_test.go @@ -2054,9 +2054,7 @@ func TestIntegrationDeleteFolderWithProvisionedDashboards(t *testing.T) { DualWriterMode: modeDw, }, }, - EnableFeatureToggles: []string{ - featuremgmt.FlagUnifiedStorageSearch, - }, + UnifiedStorageEnableSearch: true, } setupProvisioningDir(t, &ops) @@ -2163,9 +2161,7 @@ func TestIntegrationProvisionedFolderPropagatesLabelsAndAnnotations(t *testing.T DualWriterMode: mode3, }, }, - EnableFeatureToggles: []string{ - featuremgmt.FlagUnifiedStorageSearch, - }, + UnifiedStorageEnableSearch: true, } setupProvisioningDir(t, &ops) diff --git a/pkg/tests/apis/helper.go b/pkg/tests/apis/helper.go index 379e3ac9734..6087b64687d 100644 --- a/pkg/tests/apis/helper.go +++ b/pkg/tests/apis/helper.go @@ -14,6 +14,7 @@ import ( "testing" "time" + githubConnection "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/api/errors" @@ -207,6 +208,10 @@ func (c *K8sTestHelper) GetEnv() server.TestEnv { return c.env } +func (c *K8sTestHelper) SetGithubConnectionFactory(f githubConnection.GithubFactory) { + c.env.GithubConnectionFactory = f +} + func (c *K8sTestHelper) GetListenerAddress() string { return c.listenerAddress } diff --git a/pkg/tests/apis/openapi_snapshots/dashboard.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/dashboard.grafana.app-v0alpha1.json index 4634143bd45..3ed64961b1f 100644 --- a/pkg/tests/apis/openapi_snapshots/dashboard.grafana.app-v0alpha1.json +++ b/pkg/tests/apis/openapi_snapshots/dashboard.grafana.app-v0alpha1.json @@ -1830,6 +1830,22 @@ "type": "string" } }, + { + "name": "panelType", + "in": "query", + "description": "find dashboards using panels of a given plugin type", + "schema": { + "type": "string" + } + }, + { + "name": "dataSourceType", + "in": "query", + "description": "find dashboards using datasources of a given plugin type", + "schema": { + "type": "string" + } + }, { "name": "permission", "in": "query", diff --git a/pkg/tests/apis/openapi_snapshots/testdata.datasource.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/grafana-testdata-datasource.datasource.grafana.app-v0alpha1.json similarity index 96% rename from pkg/tests/apis/openapi_snapshots/testdata.datasource.grafana.app-v0alpha1.json rename to pkg/tests/apis/openapi_snapshots/grafana-testdata-datasource.datasource.grafana.app-v0alpha1.json index 4249cf16db0..a6d02c218c6 100644 --- a/pkg/tests/apis/openapi_snapshots/testdata.datasource.grafana.app-v0alpha1.json +++ b/pkg/tests/apis/openapi_snapshots/grafana-testdata-datasource.datasource.grafana.app-v0alpha1.json @@ -2,10 +2,10 @@ "openapi": "3.0.0", "info": { "description": "Generates test data in different forms", - "title": "testdata.datasource.grafana.app/v0alpha1" + "title": "grafana-testdata-datasource.datasource.grafana.app/v0alpha1" }, "paths": { - "/apis/testdata.datasource.grafana.app/v0alpha1/": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/": { "get": { "tags": [ "API Discovery" @@ -36,7 +36,7 @@ } } }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/connections/{name}/query": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/connections/{name}/query": { "post": { "tags": [ "Connections (deprecated)" @@ -68,7 +68,7 @@ "deprecated": true, "x-kubernetes-action": "connect", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "QueryDataResponse" } @@ -96,7 +96,7 @@ } ] }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources": { "get": { "tags": [ "DataSource" @@ -137,7 +137,7 @@ }, "x-kubernetes-action": "list", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "DataSource" } @@ -254,7 +254,7 @@ } ] }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}": { "get": { "tags": [ "DataSource" @@ -285,7 +285,7 @@ }, "x-kubernetes-action": "get", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "DataSource" } @@ -322,7 +322,7 @@ } ] }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}/health": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}/health": { "get": { "tags": [ "DataSource" @@ -343,7 +343,7 @@ }, "x-kubernetes-action": "connect", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "HealthCheckResult" } @@ -371,7 +371,7 @@ } ] }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}/query": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}/query": { "post": { "tags": [ "DataSource" @@ -401,7 +401,7 @@ }, "x-kubernetes-action": "connect", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "QueryDataResponse" } @@ -429,7 +429,7 @@ } ] }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}/resource": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/datasources/{name}/resource": { "get": { "tags": [ "DataSource" @@ -450,7 +450,7 @@ }, "x-kubernetes-action": "connect", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "Status" } @@ -478,7 +478,7 @@ } ] }, - "/apis/testdata.datasource.grafana.app/v0alpha1/namespaces/{namespace}/queryconvert/{name}": { + "/apis/grafana-testdata-datasource.datasource.grafana.app/v0alpha1/namespaces/{namespace}/queryconvert/{name}": { "post": { "tags": [ "QueryDataRequest" @@ -499,7 +499,7 @@ }, "x-kubernetes-action": "connect", "x-kubernetes-group-version-kind": { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "version": "v0alpha1", "kind": "QueryDataRequest" } @@ -620,7 +620,7 @@ "apiVersion": { "type": "string", "enum": [ - "testdata.datasource.grafana.app/v0alpha1" + "grafana-testdata-datasource.datasource.grafana.app/v0alpha1" ] }, "kind": { @@ -660,7 +660,7 @@ }, "x-kubernetes-group-version-kind": [ { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "kind": "DataSource", "version": "v0alpha1" } @@ -703,7 +703,7 @@ }, "x-kubernetes-group-version-kind": [ { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "kind": "DataSourceList", "version": "v0alpha1" } @@ -744,7 +744,7 @@ }, "x-kubernetes-group-version-kind": [ { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "kind": "HealthCheckResult", "version": "v0alpha1" } @@ -833,7 +833,7 @@ }, "x-kubernetes-group-version-kind": [ { - "group": "testdata.datasource.grafana.app", + "group": "grafana-testdata-datasource.datasource.grafana.app", "kind": "QueryDataResponse", "version": "v0alpha1" } diff --git a/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json index 0e2f06946ac..fc8efbaabbb 100644 --- a/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json +++ b/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json @@ -4559,7 +4559,7 @@ } ] }, - "webhook": { + "token": { "description": "Token is the reference of the token used to act as the Connection. This value is stored securely and cannot be read back", "default": {}, "allOf": [ diff --git a/pkg/tests/apis/openapi_test.go b/pkg/tests/apis/openapi_test.go index aeb629a5939..d73463a7daf 100644 --- a/pkg/tests/apis/openapi_test.go +++ b/pkg/tests/apis/openapi_test.go @@ -124,7 +124,7 @@ func TestIntegrationOpenAPIs(t *testing.T) { Group: "shorturl.grafana.app", Version: "v1beta1", }, { - Group: "testdata.datasource.grafana.app", + Group: "grafana-testdata-datasource.datasource.grafana.app", Version: "v0alpha1", }, { Group: "logsdrilldown.grafana.app", diff --git a/pkg/tests/apis/provisioning/connection_repositories_test.go b/pkg/tests/apis/provisioning/connection_repositories_test.go index e6ef823b801..4def16377e3 100644 --- a/pkg/tests/apis/provisioning/connection_repositories_test.go +++ b/pkg/tests/apis/provisioning/connection_repositories_test.go @@ -2,13 +2,13 @@ package provisioning import ( "context" + "encoding/base64" "encoding/json" "net/http" "testing" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" @@ -20,7 +20,7 @@ func TestIntegrationProvisioning_ConnectionRepositories(t *testing.T) { helper := runGrafana(t) ctx := context.Background() - createOptions := metav1.CreateOptions{FieldValidation: "Strict"} + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) // Create a connection for testing connection := &unstructured.Unstructured{Object: map[string]any{ @@ -39,13 +39,12 @@ func TestIntegrationProvisioning_ConnectionRepositories(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} - - _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) - require.NoError(t, err, "failed to create connection") + _, err := helper.CreateGithubConnection(t, ctx, connection) + require.NoError(t, err) t.Run("endpoint returns not implemented", func(t *testing.T) { var statusCode int @@ -129,14 +128,14 @@ func TestIntegrationProvisioning_ConnectionRepositoriesResponseType(t *testing.T helper := runGrafana(t) ctx := context.Background() - createOptions := metav1.CreateOptions{FieldValidation: "Strict"} + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) // Create a connection for testing connection := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "provisioning.grafana.app/v0alpha1", "kind": "Connection", "metadata": map[string]any{ - "name": "connection-repositories-type-test", + "name": "connection-repositories-test", "namespace": "default", }, "spec": map[string]any{ @@ -148,13 +147,12 @@ func TestIntegrationProvisioning_ConnectionRepositoriesResponseType(t *testing.T }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} - - _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) - require.NoError(t, err, "failed to create connection") + _, err := helper.CreateGithubConnection(t, ctx, connection) + require.NoError(t, err) t.Run("verify ExternalRepositoryList type exists in API", func(t *testing.T) { // Verify the type is registered and can be instantiated diff --git a/pkg/tests/apis/provisioning/connection_status_auth_test.go b/pkg/tests/apis/provisioning/connection_status_auth_test.go index fbddd85999a..0deaa3eeefe 100644 --- a/pkg/tests/apis/provisioning/connection_status_auth_test.go +++ b/pkg/tests/apis/provisioning/connection_status_auth_test.go @@ -2,12 +2,12 @@ package provisioning import ( "context" + "encoding/base64" "net/http" "testing" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/grafana/grafana/pkg/util/testutil" @@ -18,7 +18,7 @@ func TestIntegrationProvisioning_ConnectionStatusAuthorization(t *testing.T) { helper := runGrafana(t) ctx := context.Background() - createOptions := metav1.CreateOptions{FieldValidation: "Strict"} + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) // Create a connection for testing connection := &unstructured.Unstructured{Object: map[string]any{ @@ -37,13 +37,12 @@ func TestIntegrationProvisioning_ConnectionStatusAuthorization(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} - - _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) - require.NoError(t, err, "failed to create connection") + _, err := helper.CreateGithubConnection(t, ctx, connection) + require.NoError(t, err) t.Run("admin can GET connection status", func(t *testing.T) { var statusCode int diff --git a/pkg/tests/apis/provisioning/connection_test.go b/pkg/tests/apis/provisioning/connection_test.go index 99f32dffa93..98418f7b54e 100644 --- a/pkg/tests/apis/provisioning/connection_test.go +++ b/pkg/tests/apis/provisioning/connection_test.go @@ -2,11 +2,20 @@ package provisioning import ( "context" + "encoding/base64" + "encoding/json" "errors" + "fmt" + "net/http" "testing" "time" + "github.com/golang-jwt/jwt/v4" + "github.com/google/go-github/v70/github" + githubConnection "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" + "github.com/grafana/grafana/pkg/extensions" "github.com/grafana/grafana/pkg/util/testutil" + ghmock "github.com/migueleliasweb/go-github-mock/src/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,12 +26,55 @@ import ( clientset "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned" ) +//nolint:gosec // Test RSA private key (generated for testing purposes only) +const testPrivateKeyPEM = `-----BEGIN RSA PRIVATE KEY----- +MIIEoQIBAAKCAQBn1MuM5hIfH6d3TNStI1ofWv/gcjQ4joi9cFijEwVLuPYkF1nD +KkSbaMGFUWiOTaB/H9fxmd/V2u04NlBY3av6m5T/sHfVSiEWAEUblh3cA34HVCmD +cqyyVty5HLGJJlSs2C7W2x7yUc9ImzyDBsyjpKOXuojJ9wN9a17D2cYU5WkXjoDC +4BHid61jn9WBTtPZXSgOdirwahNzxZQSIP7DA9T8yiZwIWPp5YesgsAPyQLCFPgM +s77xz/CEUnEYQ35zI/k/mQrwKdQ/ZP8xLwQohUID0BIxE7G5quL069RuuCZWZkoF +oPiZbp7HSryz1+19jD3rFT7eHGUYvAyCnXmXAgMBAAECggEADSs4Bc7ITZo+Kytb +bfol3AQ2n8jcRrANN7mgBE7NRSVYUouDnvUlbnCC2t3QXPwLdxQa11GkygLSQ2bg +GeVDgq1o4GUJTcvxFlFCcpU/hEANI/DQsxNAQ/4wUGoLOlHaO3HPvwBblHA70gGe +Ux/xpG+lMAFAiB0EHEwZ4M0mClBEOQv3NzaFTWuBHtIMS8eid7M1q5qz9+rCgZSL +KBBHo0OvUbajG4CWl8SM6LUYapASGg+U17E+4xA3npwpIdsk+CbtX+vvX324n4kn +0EkrJqCjv8M1KiCKAP+UxwP00ywxOg4PN+x+dHI/I7xBvEKe/x6BltVSdGA+PlUK +02wagQKBgQDF7gdQLFIagPH7X7dBP6qEGxj/Ck9Qdz3S1gotPkVeq+1/UtQijYZ1 +j44up/0yB2B9P4kW091n+iWcyfoU5UwBua9dHvCZP3QH05LR1ZscUHxLGjDPBASt +l2xSq0hqqNWBspb1M0eCY0Yxi65iDkj3xsI2iN35BEb1FlWdR5KGvwKBgQCGS0ce +wASWbZIPU2UoKGOQkIJU6QmLy0KZbfYkpyfE8IxGttYVEQ8puNvDDNZWHNf+LP85 +c8iV6SfnWiLmu1XkG2YmJFBCCAWgJ8Mq2XQD8E+a/xcaW3NqlcC5+I2czX367j3r +69wZSxRbzR+DCfOiIkrekJImwN183ZYy2cBbKQKBgFj86IrSMmO6H5Ft+j06u5ZD +fJyF7Rz3T3NwSgkHWzbyQ4ggHEIgsRg/36P4YSzSBj6phyAdRwkNfUWdxXMJmH+a +FU7frzqnPaqbJAJ1cBRt10QI1XLtkpDdaJVObvONTtjOC3LYiEkGCzQRYeiyFXpZ +AU51gJ8JnkFotjtNR4KPAoGAehVREDlLcl0lnN0ZZspgyPk2Im6/iOA9KTH3xBZZ +ZwWu4FIyiHA7spgk4Ep5R0ttZ9oMI3SIcw/EgONGOy8uw/HMiPwWIhEc3B2JpRiO +CU6bb7JalFFyuQBudiHoyxVcY5PVovWF31CLr3DoJr4TR9+Y5H/U/XnzYCIo+w1N +exECgYBFAGKYTIeGAvhIvD5TphLpbCyeVLBIq5hRyrdRY+6Iwqdr5PGvLPKwin5+ ++4CDhWPW4spq8MYPCRiMrvRSctKt/7FhVGL2vE/0VY3TcLk14qLC+2+0lnPVgnYn +u5/wOyuHp1cIBnjeN41/pluOWFBHI9xLW3ExLtmYMiecJ8VdRA== +-----END RSA PRIVATE KEY-----` + +//nolint:gosec // Test RSA public key (generated for testing purposes only) +const testPublicKeyPem = `-----BEGIN PUBLIC KEY----- +MIIBITANBgkqhkiG9w0BAQEFAAOCAQ4AMIIBCQKCAQBn1MuM5hIfH6d3TNStI1of +Wv/gcjQ4joi9cFijEwVLuPYkF1nDKkSbaMGFUWiOTaB/H9fxmd/V2u04NlBY3av6 +m5T/sHfVSiEWAEUblh3cA34HVCmDcqyyVty5HLGJJlSs2C7W2x7yUc9ImzyDBsyj +pKOXuojJ9wN9a17D2cYU5WkXjoDC4BHid61jn9WBTtPZXSgOdirwahNzxZQSIP7D +A9T8yiZwIWPp5YesgsAPyQLCFPgMs77xz/CEUnEYQ35zI/k/mQrwKdQ/ZP8xLwQo +hUID0BIxE7G5quL069RuuCZWZkoFoPiZbp7HSryz1+19jD3rFT7eHGUYvAyCnXmX +AgMBAAE= +-----END PUBLIC KEY-----` + func TestIntegrationProvisioning_ConnectionCRUDL(t *testing.T) { testutil.SkipIntegrationTestInShortMode(t) helper := runGrafana(t) - createOptions := metav1.CreateOptions{FieldValidation: "Strict"} ctx := context.Background() + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) + + decryptService := helper.GetEnv().DecryptService + require.NotNil(t, decryptService, "decrypt service not wired properly") t.Run("should perform CRUDL requests on connection", func(t *testing.T) { connection := &unstructured.Unstructured{Object: map[string]any{ @@ -41,12 +93,12 @@ func TestIntegrationProvisioning_ConnectionCRUDL(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} // CREATE - _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) + _, err := helper.CreateGithubConnection(t, ctx, connection) require.NoError(t, err, "failed to create resource") // READ @@ -64,6 +116,22 @@ func TestIntegrationProvisioning_ConnectionCRUDL(t *testing.T) { require.Contains(t, output.Object, "secure", "object should contain secure") assert.Contains(t, output.Object["secure"], "privateKey", "secure should contain PrivateKey") + // Verifying token + assert.Contains(t, output.Object["secure"], "token", "token should be created") + secretName, found, err := unstructured.NestedString(output.Object, "secure", "token", "name") + require.NoError(t, err, "error getting secret name") + require.True(t, found, "secret name should exist: %v", output.Object) + decrypted, err := decryptService.Decrypt(ctx, "provisioning.grafana.app", output.GetNamespace(), secretName) + require.NoError(t, err, "decryption error") + require.Len(t, decrypted, 1) + + val := decrypted[secretName].Value() + require.NotNil(t, val) + k := val.DangerouslyExposeAndConsumeValue() + valid, err := verifyToken(t, "123456", testPublicKeyPem, k) + require.NoError(t, err, "error verifying token: %s", k) + require.True(t, valid, "token should be valid: %s", k) + // LIST list, err := helper.Connections.Resource.List(ctx, metav1.ListOptions{}) require.NoError(t, err, "failed to list resource") @@ -81,22 +149,22 @@ func TestIntegrationProvisioning_ConnectionCRUDL(t *testing.T) { "spec": map[string]any{ "type": "github", "github": map[string]any{ - "appID": "456789", - "installationID": "454545", + "appID": "123456", + "installationID": "454546", }, }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} - res, err := helper.Connections.Resource.Update(ctx, updatedConnection, metav1.UpdateOptions{}) + res, err := helper.UpdateGithubConnection(t, ctx, updatedConnection) require.NoError(t, err, "failed to update resource") spec = res.Object["spec"].(map[string]any) require.Contains(t, spec, "github") githubInfo = spec["github"].(map[string]any) - assert.Equal(t, "456789", githubInfo["appID"], "appID should be updated") + assert.Equal(t, "454546", githubInfo["installationID"], "installationID should be updated") // DELETE require.NoError(t, helper.Connections.Resource.Delete(ctx, "connection", metav1.DeleteOptions{}), "failed to delete resource") @@ -122,7 +190,7 @@ func TestIntegrationProvisioning_ConnectionCRUDL(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} @@ -155,9 +223,12 @@ func TestIntegrationProvisioning_ConnectionCRUDL(t *testing.T) { } func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { + testutil.SkipIntegrationTestInShortMode(t) + helper := runGrafana(t) createOptions := metav1.CreateOptions{FieldValidation: "Strict"} ctx := context.Background() + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) t.Run("should fail when type is empty", func(t *testing.T) { connection := &unstructured.Unstructured{Object: map[string]any{ @@ -172,13 +243,13 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) require.Error(t, err, "failed to create resource") - assert.Contains(t, err.Error(), "type must be specified") + assert.Contains(t, err.Error(), "connection type \"\" is not supported") }) t.Run("should fail when type is invalid", func(t *testing.T) { @@ -194,13 +265,57 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) require.Error(t, err, "failed to create resource") - assert.Contains(t, err.Error(), "spec.type: Unsupported value: \"some-invalid-type\"") + assert.Contains(t, err.Error(), "connection type \"some-invalid-type\" is not supported") + }) + + t.Run("should fail when type is 'git'", func(t *testing.T) { + connection := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "provisioning.grafana.app/v0alpha1", + "kind": "Connection", + "metadata": map[string]any{ + "name": "connection", + "namespace": "default", + }, + "spec": map[string]any{ + "type": "git", + }, + "secure": map[string]any{ + "privateKey": map[string]any{ + "create": privateKeyBase64, + }, + }, + }} + _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) + require.Error(t, err, "failed to create resource") + assert.Contains(t, err.Error(), "connection type \"git\" is not supported") + }) + + t.Run("should fail when type is 'local'", func(t *testing.T) { + connection := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "provisioning.grafana.app/v0alpha1", + "kind": "Connection", + "metadata": map[string]any{ + "name": "connection", + "namespace": "default", + }, + "spec": map[string]any{ + "type": "local", + }, + "secure": map[string]any{ + "privateKey": map[string]any{ + "create": privateKeyBase64, + }, + }, + }} + _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) + require.Error(t, err, "failed to create resource") + assert.Contains(t, err.Error(), "connection type \"local\" is not supported") }) t.Run("should fail when type is github but 'github' field is not there", func(t *testing.T) { @@ -216,13 +331,13 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, }, }} _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) require.Error(t, err, "failed to create resource") - assert.Contains(t, err.Error(), "github info must be specified for GitHub connection") + assert.Contains(t, err.Error(), "invalid github connection") }) t.Run("should fail when type is github but private key is not there", func(t *testing.T) { @@ -246,7 +361,7 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { assert.Contains(t, err.Error(), "privateKey must be specified for GitHub connection") }) - t.Run("should fail when type is github but a client Secret is specified", func(t *testing.T) { + t.Run("should fail when type is github but a client Secret is also specified", func(t *testing.T) { connection := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "provisioning.grafana.app/v0alpha1", "kind": "Connection", @@ -263,7 +378,7 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "someSecret", + "create": privateKeyBase64, }, "clientSecret": map[string]any{ "create": "someSecret", @@ -275,6 +390,100 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { assert.Contains(t, err.Error(), "clientSecret is forbidden in GitHub connection") }) + t.Run("should fail when type is github and github API is unavailable", func(t *testing.T) { + connectionFactory := helper.GetEnv().GithubConnectionFactory.(*githubConnection.Factory) + connectionFactory.Client = ghmock.NewMockedHTTPClient( + ghmock.WithRequestMatchHandler( + ghmock.GetApp, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + require.NoError(t, json.NewEncoder(w).Encode(github.ErrorResponse{ + Response: &http.Response{ + StatusCode: http.StatusServiceUnavailable, + }, + Message: "Service unavailable", + })) + }), + ), + ) + helper.SetGithubConnectionFactory(connectionFactory) + + connection := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "provisioning.grafana.app/v0alpha1", + "kind": "Connection", + "metadata": map[string]any{ + "name": "connection", + "namespace": "default", + }, + "spec": map[string]any{ + "type": "github", + "github": map[string]any{ + "appID": "123456", + "installationID": "454545", + }, + }, + "secure": map[string]any{ + "privateKey": map[string]any{ + "create": privateKeyBase64, + }, + }, + }} + _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) + require.Error(t, err, "failed to create resource") + assert.Contains(t, err.Error(), "spec.token: Internal error: github is unavailable") + }) + + t.Run("should fail when type is github and returned app ID doesn't match given one", func(t *testing.T) { + var appID int64 = 123455 + appSlug := "appSlug" + connectionFactory := helper.GetEnv().GithubConnectionFactory.(*githubConnection.Factory) + connectionFactory.Client = ghmock.NewMockedHTTPClient( + ghmock.WithRequestMatch( + ghmock.GetApp, github.App{ + ID: &appID, + Slug: &appSlug, + }, + ), + ) + helper.SetGithubConnectionFactory(connectionFactory) + + connection := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "provisioning.grafana.app/v0alpha1", + "kind": "Connection", + "metadata": map[string]any{ + "name": "connection", + "namespace": "default", + }, + "spec": map[string]any{ + "type": "github", + "github": map[string]any{ + "appID": "123456", + "installationID": "454545", + }, + }, + "secure": map[string]any{ + "privateKey": map[string]any{ + "create": privateKeyBase64, + }, + }, + }} + _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) + require.Error(t, err, "failed to create resource") + assert.Contains(t, err.Error(), "spec.appID: Invalid value: \"123456\": appID mismatch") + }) +} + +func TestIntegrationProvisioning_ConnectionEnterpriseValidation(t *testing.T) { + testutil.SkipIntegrationTestInShortMode(t) + + if !extensions.IsEnterprise { + t.Skip("Skipping integration test when not enterprise") + } + + helper := runGrafana(t) + createOptions := metav1.CreateOptions{FieldValidation: "Strict"} + ctx := context.Background() + t.Run("should fail when type is bitbucket but 'bitbucket' field is not there", func(t *testing.T) { connection := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "provisioning.grafana.app/v0alpha1", @@ -294,7 +503,7 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { }} _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) require.Error(t, err, "failed to create resource") - assert.Contains(t, err.Error(), "bitbucket info must be specified in Bitbucket connection") + assert.Contains(t, err.Error(), "invalid bitbucket connection") }) t.Run("should fail when type is bitbucket but client secret is not there", func(t *testing.T) { @@ -364,7 +573,7 @@ func TestIntegrationProvisioning_ConnectionValidation(t *testing.T) { }} _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) require.Error(t, err, "failed to create resource") - assert.Contains(t, err.Error(), "gitlab info must be specified in Gitlab connection") + assert.Contains(t, err.Error(), "invalid gitlab connection") }) t.Run("should fail when type is gitlab but client secret is not there", func(t *testing.T) { @@ -428,6 +637,7 @@ func TestIntegrationConnectionController_HealthCheckUpdates(t *testing.T) { provisioningClient, err := clientset.NewForConfig(restConfig) require.NoError(t, err) connClient := provisioningClient.ProvisioningV0alpha1().Connections(namespace) + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) t.Run("health check gets updated after initial creation", func(t *testing.T) { // Create a connection using unstructured (like other connection tests) @@ -447,12 +657,12 @@ func TestIntegrationConnectionController_HealthCheckUpdates(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "test-private-key", + "create": privateKeyBase64, }, }, }} - createdUnstructured, err := helper.Connections.Resource.Create(ctx, connUnstructured, metav1.CreateOptions{}) + createdUnstructured, err := helper.CreateGithubConnection(t, ctx, connUnstructured) require.NoError(t, err) require.NotNil(t, createdUnstructured) @@ -501,12 +711,12 @@ func TestIntegrationConnectionController_HealthCheckUpdates(t *testing.T) { }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "test-private-key-2", + "create": privateKeyBase64, }, }, }} - createdUnstructured, err := helper.Connections.Resource.Create(ctx, connUnstructured, metav1.CreateOptions{}) + createdUnstructured, err := helper.CreateGithubConnection(t, ctx, connUnstructured) require.NoError(t, err) require.NotNil(t, createdUnstructured) @@ -538,7 +748,7 @@ func TestIntegrationConnectionController_HealthCheckUpdates(t *testing.T) { updatedUnstructured := latestUnstructured.DeepCopy() githubSpec := updatedUnstructured.Object["spec"].(map[string]any)["github"].(map[string]any) githubSpec["appID"] = "99999" - _, err = helper.Connections.Resource.Update(ctx, updatedUnstructured, metav1.UpdateOptions{}) + _, err = helper.UpdateGithubConnection(t, ctx, updatedUnstructured) require.NoError(t, err) // Wait for reconciliation after spec change @@ -566,6 +776,7 @@ func TestIntegrationProvisioning_RepositoryFieldSelectorByConnection(t *testing. helper := runGrafana(t) ctx := context.Background() createOptions := metav1.CreateOptions{FieldValidation: "Strict"} + privateKeyBase64 := base64.StdEncoding.EncodeToString([]byte(testPrivateKeyPEM)) // Create a connection first connection := &unstructured.Unstructured{Object: map[string]any{ @@ -584,12 +795,12 @@ func TestIntegrationProvisioning_RepositoryFieldSelectorByConnection(t *testing. }, "secure": map[string]any{ "privateKey": map[string]any{ - "create": "test-private-key", + "create": privateKeyBase64, }, }, }} - _, err := helper.Connections.Resource.Create(ctx, connection, createOptions) + _, err := helper.CreateGithubConnection(t, ctx, connection) require.NoError(t, err, "failed to create connection") t.Cleanup(func() { @@ -731,3 +942,27 @@ func TestIntegrationProvisioning_RepositoryFieldSelectorByConnection(t *testing. assert.Contains(t, names, "repo-with-different-connection") }) } + +func verifyToken(t *testing.T, appID, publicKey, token string) (bool, error) { + t.Helper() + + // Parse the private key + key, err := jwt.ParseRSAPublicKeyFromPEM([]byte(publicKey)) + if err != nil { + return false, err + } + + parsedToken, err := jwt.Parse(token, func(token *jwt.Token) (any, error) { + return key, nil + }, jwt.WithValidMethods([]string{jwt.SigningMethodRS256.Alg()})) + if err != nil { + return false, err + } + + claims, ok := parsedToken.Claims.(jwt.MapClaims) + if !ok || !parsedToken.Valid { + return false, fmt.Errorf("invalid token") + } + + return claims.VerifyIssuer(appID, true), nil +} diff --git a/pkg/tests/apis/provisioning/helper_test.go b/pkg/tests/apis/provisioning/helper_test.go index 791ac4b8a20..34b57afe3c6 100644 --- a/pkg/tests/apis/provisioning/helper_test.go +++ b/pkg/tests/apis/provisioning/helper_test.go @@ -10,11 +10,14 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "testing" "text/template" "time" + "github.com/google/go-github/v70/github" + "github.com/grafana/grafana/pkg/extensions" ghmock "github.com/migueleliasweb/go-github-mock/src/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,6 +33,7 @@ import ( dashboardsV2beta1 "github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/v2beta1" folder "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1" provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1" + githubConnection "github.com/grafana/grafana/apps/provisioning/pkg/connection/github" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -699,13 +703,18 @@ func runGrafana(t *testing.T, options ...grafanaOption) *provisioningTestHelper // (instance is needed for export jobs, folder for most operations) ProvisioningAllowedTargets: []string{"folder", "instance"}, } + + if extensions.IsEnterprise { + opts.ProvisioningRepositoryTypes = []string{"local", "github", "gitlab", "bitbucket"} + } + for _, o := range options { o(&opts) } helper := apis.NewK8sTestHelper(t, opts) - // FIXME: keeping this line here to keep the dependency around until we have tests which use this again. - helper.GetEnv().GitHubFactory.Client = ghmock.NewMockedHTTPClient() + // FIXME: keeping these lines here to keep the dependency around until we have tests which use this again. + helper.GetEnv().GithubRepoFactory.Client = ghmock.NewMockedHTTPClient() repositories := helper.GetResourceClient(apis.ResourceClientArgs{ User: helper.Org1.Admin, @@ -973,6 +982,79 @@ func (h *provisioningTestHelper) CleanupAllRepos(t *testing.T) { }, waitTimeoutDefault, waitIntervalDefault, "repositories should be cleaned up between subtests") } +func (h *provisioningTestHelper) CreateGithubConnection( + t *testing.T, + ctx context.Context, + connection *unstructured.Unstructured, +) (*unstructured.Unstructured, error) { + t.Helper() + + err := h.setGithubClient(t, connection) + if err != nil { + return nil, err + } + + return h.Connections.Resource.Create(ctx, connection, metav1.CreateOptions{FieldValidation: "Strict"}) +} + +func (h *provisioningTestHelper) UpdateGithubConnection( + t *testing.T, + ctx context.Context, + connection *unstructured.Unstructured, +) (*unstructured.Unstructured, error) { + t.Helper() + + err := h.setGithubClient(t, connection) + if err != nil { + return nil, err + } + + return h.Connections.Resource.Update(ctx, connection, metav1.UpdateOptions{FieldValidation: "Strict"}) +} + +func (h *provisioningTestHelper) setGithubClient(t *testing.T, connection *unstructured.Unstructured) error { + t.Helper() + + objectSpec := connection.Object["spec"].(map[string]interface{}) + githubObj := objectSpec["github"].(map[string]interface{}) + appID := githubObj["appID"].(string) + id, err := strconv.ParseInt(appID, 10, 64) + if err != nil { + return err + } + + appSlug := "someSlug" + connectionFactory := h.GetEnv().GithubConnectionFactory.(*githubConnection.Factory) + connectionFactory.Client = ghmock.NewMockedHTTPClient( + ghmock.WithRequestMatchHandler( + ghmock.GetApp, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + app := github.App{ + ID: &id, + Slug: &appSlug, + } + _, _ = w.Write(ghmock.MustMarshal(app)) + }), + ), + ghmock.WithRequestMatchHandler( + ghmock.GetAppInstallationsByInstallationId, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("installation_id") + idInt, _ := strconv.ParseInt(id, 10, 64) + w.WriteHeader(http.StatusOK) + installation := github.Installation{ + ID: &idInt, + } + _, _ = w.Write(ghmock.MustMarshal(installation)) + }), + ), + ) + h.SetGithubConnectionFactory(connectionFactory) + + return nil +} + func postHelper(t *testing.T, helper apis.K8sTestHelper, path string, body interface{}, user apis.User) (map[string]interface{}, int, error) { return requestHelper(t, helper, http.MethodPost, path, body, user) } diff --git a/pkg/tests/apis/provisioning/repository_test.go b/pkg/tests/apis/provisioning/repository_test.go index f2447e71d23..0f3509602b7 100644 --- a/pkg/tests/apis/provisioning/repository_test.go +++ b/pkg/tests/apis/provisioning/repository_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/extensions" provisioningAPIServer "github.com/grafana/grafana/pkg/registry/apis/provisioning" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -149,10 +150,19 @@ func TestIntegrationProvisioning_CreatingAndGetting(t *testing.T) { } } - assert.ElementsMatch(collect, []provisioning.RepositoryType{ - provisioning.LocalRepositoryType, - provisioning.GitHubRepositoryType, - }, settings.AvailableRepositoryTypes) + if extensions.IsEnterprise { + assert.ElementsMatch(collect, []provisioning.RepositoryType{ + provisioning.LocalRepositoryType, + provisioning.GitHubRepositoryType, + provisioning.BitbucketRepositoryType, + provisioning.GitLabRepositoryType, + }, settings.AvailableRepositoryTypes) + } else { + assert.ElementsMatch(collect, []provisioning.RepositoryType{ + provisioning.LocalRepositoryType, + provisioning.GitHubRepositoryType, + }, settings.AvailableRepositoryTypes) + } }, time.Second*10, time.Millisecond*100, "Expected settings to match") }) diff --git a/pkg/tests/testinfra/testinfra.go b/pkg/tests/testinfra/testinfra.go index 88f65223675..81e243f42b5 100644 --- a/pkg/tests/testinfra/testinfra.go +++ b/pkg/tests/testinfra/testinfra.go @@ -622,6 +622,12 @@ func CreateGrafDir(t *testing.T, opts GrafanaOpts) (string, string) { _, err = provisioningSect.NewKey("allowed_targets", strings.Join(opts.ProvisioningAllowedTargets, "|")) require.NoError(t, err) } + if len(opts.ProvisioningRepositoryTypes) > 0 { + provisioningSect, err := getOrCreateSection("provisioning") + require.NoError(t, err) + _, err = provisioningSect.NewKey("repository_types", strings.Join(opts.ProvisioningRepositoryTypes, "|")) + require.NoError(t, err) + } if opts.EnableSCIM { scimSection, err := getOrCreateSection("auth.scim") require.NoError(t, err) @@ -731,6 +737,7 @@ type GrafanaOpts struct { UnifiedStorageMaxPageSizeBytes int PermittedProvisioningPaths string ProvisioningAllowedTargets []string + ProvisioningRepositoryTypes []string GrafanaComSSOAPIToken string LicensePath string EnableRecordingRules bool diff --git a/public/app/core/icons/cached.json b/public/app/core/icons/cached.json index 6e35e64dd0c..9987d867738 100644 --- a/public/app/core/icons/cached.json +++ b/public/app/core/icons/cached.json @@ -29,6 +29,7 @@ "unicons/bookmark", "unicons/book-open", "unicons/brackets-curly", + "unicons/brain", "unicons/bug", "unicons/building", "unicons/calculator-alt", diff --git a/public/app/features/apiserver/client.test.ts b/public/app/features/apiserver/client.test.ts index 01787b4a18d..02f6463b8f0 100644 --- a/public/app/features/apiserver/client.test.ts +++ b/public/app/features/apiserver/client.test.ts @@ -28,18 +28,17 @@ describe('DatasourceAPIVersions', () => { it('get', async () => { const getMock = jest.fn().mockResolvedValue({ groups: [ - { name: 'testdata.datasource.grafana.app', preferredVersion: { version: 'v1' } }, + { name: 'grafana-testdata-datasource.datasource.grafana.app', preferredVersion: { version: 'v1' } }, { name: 'prometheus.datasource.grafana.app', preferredVersion: { version: 'v2' } }, { name: 'myorg-myplugin.datasource.grafana.app', preferredVersion: { version: 'v3' } }, ], }); getBackendSrv().get = getMock; const apiVersions = new DatasourceAPIVersions(); - expect(await apiVersions.get('testdata')).toBe('v1'); expect(await apiVersions.get('grafana-testdata-datasource')).toBe('v1'); expect(await apiVersions.get('prometheus')).toBe('v2'); expect(await apiVersions.get('graphite')).toBeUndefined(); - expect(await apiVersions.get('myorg-myplugin-datasource')).toBe('v3'); + expect(await apiVersions.get('myorg-myplugin')).toBe('v3'); expect(getMock).toHaveBeenCalledTimes(1); expect(getMock).toHaveBeenCalledWith('/apis'); }); diff --git a/public/app/features/apiserver/client.ts b/public/app/features/apiserver/client.ts index 2907fc9e80b..7966d0fa56b 100644 --- a/public/app/features/apiserver/client.ts +++ b/public/app/features/apiserver/client.ts @@ -162,17 +162,6 @@ export class DatasourceAPIVersions { if (group.name.includes('datasource.grafana.app')) { const id = group.name.split('.')[0]; apiVersions[id] = group.preferredVersion.version; - // workaround for plugins that don't append '-datasource' for the group name - // e.g. org-plugin-datasource uses org-plugin.datasource.grafana.app - if (!id.endsWith('-datasource')) { - if (!id.includes('-')) { - // workaroud for Grafana plugins that don't include the org either - // e.g. testdata uses testdata.datasource.grafana.app - apiVersions[`grafana-${id}-datasource`] = group.preferredVersion.version; - } else { - apiVersions[`${id}-datasource`] = group.preferredVersion.version; - } - } } }); this.apiVersions = apiVersions; diff --git a/public/app/features/browse-dashboards/components/RecentlyViewedDashboards.tsx b/public/app/features/browse-dashboards/components/RecentlyViewedDashboards.tsx index effa0c8009b..f0c2d4d3a13 100644 --- a/public/app/features/browse-dashboards/components/RecentlyViewedDashboards.tsx +++ b/public/app/features/browse-dashboards/components/RecentlyViewedDashboards.tsx @@ -4,6 +4,7 @@ import { useAsyncRetry } from 'react-use'; import { GrafanaTheme2, store } from '@grafana/data'; import { t, Trans } from '@grafana/i18n'; +import { reportInteraction } from '@grafana/runtime'; import { evaluateBooleanFlag } from '@grafana/runtime/internal'; import { Button, CollapsableSection, Spinner, Stack, Text, useStyles2, Grid } from '@grafana/ui'; import { contextSrv } from 'app/core/services/context_srv'; @@ -35,10 +36,18 @@ export function RecentlyViewedDashboards() { const { foldersByUid } = useDashboardLocationInfo(recentDashboards.length > 0); const handleClearHistory = () => { + reportInteraction('grafana_recently_viewed_dashboards_clear_history'); store.set(recentDashboardsKey, JSON.stringify([])); retry(); }; + const handleSectionToggle = () => { + reportInteraction('grafana_recently_viewed_dashboards_toggle_section', { + expanded: !isOpen, + }); + setIsOpen(!isOpen); + }; + if (!evaluateBooleanFlag('recentlyViewedDashboards', false) || recentDashboards.length === 0) { return null; } @@ -48,7 +57,7 @@ export function RecentlyViewedDashboards() { headerDataTestId="browseDashboardsRecentlyViewedTitle" label={ - setIsOpen(!isOpen)}> + Recently viewed