diff --git a/.betterer.results b/.betterer.results index 31d1f23e1c8..5b5be31e246 100644 --- a/.betterer.results +++ b/.betterer.results @@ -3513,8 +3513,7 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use export all (\`export * from ...\`)", "2"], [0, 0, 0, "Do not use export all (\`export * from ...\`)", "3"], [0, 0, 0, "Do not use export all (\`export * from ...\`)", "4"], - [0, 0, 0, "Do not use export all (\`export * from ...\`)", "5"], - [0, 0, 0, "Do not use export all (\`export * from ...\`)", "6"] + [0, 0, 0, "Do not use export all (\`export * from ...\`)", "5"] ], "public/app/features/datasources/state/navModel.ts:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], @@ -4770,8 +4769,7 @@ exports[`better eslint`] = { [0, 0, 0, "No untranslated strings. Wrap text with ", "35"], [0, 0, 0, "No untranslated strings. Wrap text with ", "36"], [0, 0, 0, "No untranslated strings. Wrap text with ", "37"], - [0, 0, 0, "No untranslated strings. Wrap text with ", "38"], - [0, 0, 0, "Styles should be written using objects.", "39"] + [0, 0, 0, "No untranslated strings. Wrap text with ", "38"] ], "public/app/features/query/state/DashboardQueryRunner/AnnotationsQueryRunner.ts:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"] @@ -7310,24 +7308,6 @@ exports[`no gf-form usage`] = { "public/app/features/query/components/QueryEditorRow.tsx:5381": [ [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"] ], - "public/app/features/query/components/QueryGroupOptions.tsx:5381": [ - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"], - [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"] - ], "public/app/features/variables/adhoc/picker/AdHocFilter.tsx:5381": [ [0, 0, 0, "gf-form usage has been deprecated. Use a component from @grafana/ui or custom CSS instead.", "5381"] ], diff --git a/.bra.toml b/.bra.toml index b32010204b6..6dc5907da11 100644 --- a/.bra.toml +++ b/.bra.toml @@ -16,7 +16,7 @@ watch_exts = [".go", ".ini", ".toml", ".template.html"] ignore_files = [".*_gen.go"] build_delay = 1500 cmds = [ - ["GO_BUILD_DEV=1", "make", "build-go"], + ["GO_BUILD_DEV=1", "make", "build-go-fast"], ["make", "gen-jsonnet"], ["./bin/grafana", "server", "-profile", "-profile-addr=127.0.0.1", "-profile-port=6000", "-profile-block-rate=1", "-profile-mutex-rate=5", "-packaging=dev", "cfg:app_mode=development"] ] diff --git a/.drone.yml b/.drone.yml index 48d7046793b..4d7826a65a7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -71,18 +71,10 @@ steps: - echo $DRONE_RUNNER_NAME image: alpine:3.20.3 name: identify-runner -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.23.1-alpine - name: compile-build-cmd - commands: - go install github.com/bazelbuild/buildtools/buildifier@latest - buildifier --lint=warn -mode=check -r . - depends_on: - - compile-build-cmd + depends_on: [] image: golang:1.23.1-alpine name: lint-starlark trigger: @@ -1278,13 +1270,6 @@ platform: os: linux services: [] steps: -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.23.1-alpine - name: compile-build-cmd - commands: - apt-get update -yq && apt-get install shellcheck - shellcheck -e SC1071 -e SC2162 scripts/**/*.sh @@ -3434,31 +3419,32 @@ steps: - |2- bash -c ' + IMAGE_TAG=$(echo "$${TAG}" | sed -e "s/+/-/g") debug= if [[ -n $${DRY_RUN} ]]; then debug=echo; fi docker login -u $${DOCKER_USER} -p $${DOCKER_PASSWORD} # Push the grafana-image-tags images - $$debug docker push grafana/grafana-image-tags:$${TAG}-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 + $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests - $$debug docker manifest push grafana/grafana:$${TAG} - $$debug docker manifest push grafana/grafana:$${TAG}-ubuntu + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG}-ubuntu # if LATEST is set, then also create & push latest if [[ -n $${LATEST} ]]; then - $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 $$debug docker manifest push grafana/grafana:latest $$debug docker manifest push grafana/grafana:latest-ubuntu @@ -3565,31 +3551,32 @@ steps: - |2- bash -c ' + IMAGE_TAG=$(echo "$${TAG}" | sed -e "s/+/-/g") debug= if [[ -n $${DRY_RUN} ]]; then debug=echo; fi docker login -u $${DOCKER_USER} -p $${DOCKER_PASSWORD} # Push the grafana-image-tags images - $$debug docker push grafana/grafana-image-tags:$${TAG}-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 - $$debug docker push grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 + $$debug docker push grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Create the grafana manifests - $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 + $$debug docker manifest create grafana/grafana:${TAG} grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 - $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:${TAG}-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 # Push the grafana manifests - $$debug docker manifest push grafana/grafana:$${TAG} - $$debug docker manifest push grafana/grafana:$${TAG}-ubuntu + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG} + $$debug docker manifest push grafana/grafana:$${IMAGE_TAG}-ubuntu # if LATEST is set, then also create & push latest if [[ -n $${LATEST} ]]; then - $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${TAG}-amd64 grafana/grafana-image-tags:$${TAG}-arm64 grafana/grafana-image-tags:$${TAG}-armv7 - $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${TAG}-ubuntu-armv7 + $$debug docker manifest create grafana/grafana:latest grafana/grafana-image-tags:$${IMAGE_TAG}-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-armv7 + $$debug docker manifest create grafana/grafana:latest-ubuntu grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-amd64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-arm64 grafana/grafana-image-tags:$${IMAGE_TAG}-ubuntu-armv7 $$debug docker manifest push grafana/grafana:latest $$debug docker manifest push grafana/grafana:latest-ubuntu @@ -3681,7 +3668,8 @@ steps: image: golang:1.23.1-alpine name: compile-build-cmd - commands: - - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + - ./bin/build artifacts packages --artifacts-editions=oss --tag $${DRONE_TAG} --src-bucket + $${PRERELEASE_BUCKET} depends_on: - compile-build-cmd environment: @@ -3691,19 +3679,6 @@ steps: from_secret: prerelease_bucket image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts -- commands: - - ./bin/build artifacts static-assets --tag ${DRONE_TAG} --static-asset-editions=grafana-oss - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_grafanauploads_base64 - PRERELEASE_BUCKET: - from_secret: prerelease_bucket - STATIC_ASSET_EDITIONS: - from_secret: static_asset_editions - image: grafana/grafana-ci-deploy:1.3.3 - name: publish-static-assets - commands: - ./bin/build artifacts storybook --tag ${DRONE_TAG} depends_on: @@ -3723,7 +3698,6 @@ steps: -f latest=$${LATEST} --repo=grafana/grafana release-pr.yml depends_on: - publish-artifacts - - publish-static-assets environment: GH_CLI_URL: https://github.com/cli/cli/releases/download/v2.50.0/gh_2.50.0_linux_amd64.tar.gz GITHUB_TOKEN: @@ -6013,6 +5987,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: e618274ea7a8bfbf3d5e151d459348aa9382fe63fe7fef76c997db3cba74779f +hmac: dc30a3a00ee542fb289da36ef6db4274684db4533c472f7f903468919d1046ac ... diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d9c88232b09..bfecf44889b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -241,7 +241,6 @@ /devenv/docker/rpmtest/ @grafana/grafana-backend-services-squad /devenv/jsonnet/ @grafana/dataviz-squad /devenv/local-npm/ @grafana/frontend-ops -/devenv/vscode/ @grafana/frontend-ops /devenv/setup.sh @grafana/grafana-backend-services-squad /devenv/plugins.yaml @grafana/plugins-platform-frontend diff --git a/.vscode/launch.json b/.vscode/launch.json index 5cfa698f5f0..28499f1fb2e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,12 @@ "program": "${workspaceFolder}/pkg/cmd/grafana/", "env": {}, "cwd": "${workspaceFolder}", - "args": ["server", "--homepath", "${workspaceFolder}", "--packaging", "dev", "cfg:app_mode=development"] + "args": [ + "server", + "--homepath", "${workspaceFolder}", + "--packaging", "dev", + "cfg:app_mode=development", + ] }, { "name": "Run API Server (testdata)", diff --git a/Makefile b/Makefile index 9c8f178a3d9..f38835b5c60 100644 --- a/Makefile +++ b/Makefile @@ -174,12 +174,16 @@ gen-jsonnet: go generate ./devenv/jsonnet .PHONY: update-workspace -update-workspace: +update-workspace: gen-go @echo "updating workspace" bash scripts/go-workspace/update-workspace.sh .PHONY: build-go build-go: gen-go update-workspace ## Build all Go binaries. + @echo "build go files with updated workspace" + $(GO) run build.go $(GO_BUILD_FLAGS) build + +build-go-fast: gen-go ## Build all Go binaries. @echo "build go files" $(GO) run build.go $(GO_BUILD_FLAGS) build diff --git a/README.md b/README.md index f111b8c58cb..7322e6a1c7d 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ If you're interested in contributing to the Grafana project: ## Get involved -- Follow [@grafana on Twitter](https://twitter.com/grafana/). +- Follow [@grafana on X (formerly Twitter)](https://x.com/grafana/). - Read and subscribe to the [Grafana blog](https://grafana.com/blog/). - If you have a specific question, check out our [discussion forums](https://community.grafana.com/). - For general discussions, join us on the [official Slack](https://slack.grafana.com) team. diff --git a/conf/defaults.ini b/conf/defaults.ini index 662e300834f..8daf5ef4a88 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -553,7 +553,7 @@ token_expiration_day_limit = # Login cookie name login_cookie_name = grafana_session -# Disable usage of Grafana build-in login solution. +# Disable usage of Grafana's built-in login solution. disable_login = false # The maximum lifetime (duration) an authenticated user can be inactive before being required to login at next visit. Default is 7 days (7d). This setting should be expressed as a duration, e.g. 5m (minutes), 6h (hours), 10d (days), 2w (weeks), 1M (month). The lifetime resets at each successful token rotation (token_rotation_interval_minutes). diff --git a/devenv/README.md b/devenv/README.md index a07506e2516..cc37e031d0a 100644 --- a/devenv/README.md +++ b/devenv/README.md @@ -81,7 +81,7 @@ host = "localhost:1025" You can access the web UI at http://localhost:12080/#/ ## Debugging setup in VS Code -An example of launch.json is provided in `devenv/vscode/launch.json`. It basically does what Makefile and .bra.toml do. The 'program' field is set to the folder name so VS Code loads all *.go files in it instead of just main.go. +An example of launch.json is provided in `.vscode/launch.json`. It basically does what Makefile and .bra.toml do. The 'program' field is set to the folder name so VS Code loads all *.go files in it instead of just main.go. ## Troubleshooting diff --git a/devenv/vscode/launch.json b/devenv/vscode/launch.json deleted file mode 100644 index d988713128a..00000000000 --- a/devenv/vscode/launch.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "grafana-server", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/pkg/cmd/grafana-server", - "env": {}, - "args": [ - "--homepath=${workspaceFolder}", - "--packaging=dev", - "cfg:app_mode=development", - ] - } - ] -} \ No newline at end of file diff --git a/docs/sources/administration/roles-and-permissions/access-control/custom-role-actions-scopes/index.md b/docs/sources/administration/roles-and-permissions/access-control/custom-role-actions-scopes/index.md index 7f3f60f1c96..1276f049da2 100644 --- a/docs/sources/administration/roles-and-permissions/access-control/custom-role-actions-scopes/index.md +++ b/docs/sources/administration/roles-and-permissions/access-control/custom-role-actions-scopes/index.md @@ -199,6 +199,26 @@ The following list contains role-based access control actions used by Grafana Ad | `grafana‑adaptive‑metrics‑app.exemptions:read` | None | Read recommendation exemptions. | | `grafana‑adaptive‑metrics‑app.exemptions:write` | None | Create, update, and delete recommendation exemptions. | +### Grafana Alerting Notification action definitions + +To enable these permissions, enable the `alertingApiServer` feature toggle. + +| Action | Applicable scopes | Description | +| -------------------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `alert.notifications.receivers:read` | `receivers:*`
`receivers:uid:*` | Read contact points. None | +| `alert.notifications.receivers.secrets:read` | `receivers:*`
`receivers:uid:*` | Export contact points with decrypted secrets.None | +| `alert.notifications.receivers:create` | None | Create a new contact points. The creator is automatically granted full access to the created contact point.None | +| `alert.notifications.receivers:write` | `receivers:*`
`receivers:uid:*` | Update existing contact points.None | +| `alert.notifications.receivers:delete` | `receivers:*`
`receivers:uid:*` | Update and delete existing contact points.None | +| `receivers.permissions:read` | `receivers:*`
`receivers:uid:*` | Read permissions for contact points.None | +| `receivers.permissions:write` | `receivers:*`
`receivers:uid:*` | Manage permissions for contact points.None | +| `alert.notifications.time-intervals:read` | None | Read mute time intervals.None | +| `alert.notifications.time-intervals:write` | None | Create new or update existing mute time intervals.None | +| `alert.notifications.time-intervals:delete` | None | Delete existing time intervals.None | +| `alert.notifications.templates:read` | None | Read templates. | +| `alert.notifications.templates:write` | None | Create new or update existing templates.None | +| `alert.notifications.templates:delete` | None | Delete existing templates.None | + ## Scope definitions The following list contains role-based access control scopes. diff --git a/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt.md b/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt.md index 0f9567a0a21..666ea653e18 100644 --- a/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt.md +++ b/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt.md @@ -149,8 +149,8 @@ Each alert instance in the `alerts` array has the following fields. | generatorURL | string | URL of the alert rule in the Grafana UI | | fingerprint | string | The labels fingerprint, alarms with the same labels will have the same fingerprint | | silenceURL | string | URL to silence the alert rule in the Grafana UI | -| dashboardURL | string | **Deprecated. It will be removed in a future release.** | -| panelURL | string | **Deprecated. It will be removed in a future release.** | +| dashboardURL | string | A link to the Grafana Dashboard if the alert has a Dashboard UID annotation | +| panelURL | string | A link to the panel if the alert has a Panel ID annotation | | imageURL | string | URL of a screenshot of a panel assigned to the rule that created this notification | {{< admonition type="note" >}} diff --git a/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier.md b/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier.md index 17468cac706..a1329970152 100644 --- a/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier.md +++ b/docs/sources/alerting/configure-notifications/manage-contact-points/integrations/webhook-notifier.md @@ -134,8 +134,8 @@ The webhook notification is a simple way to send information about a state chang | generatorURL | string | URL of the alert rule in the Grafana UI | | fingerprint | string | The labels fingerprint, alarms with the same labels will have the same fingerprint | | silenceURL | string | URL to silence the alert rule in the Grafana UI | -| dashboardURL | string | **Will be deprecated soon** | -| panelURL | string | **Will be deprecated soon** | +| dashboardURL | string | A link to the Grafana Dashboard if the alert has a Dashboard UID annotation | +| panelURL | string | A link to the panel if the alert has a Panel ID annotation | | imageURL | string | URL of a screenshot of a panel assigned to the rule that created this notification | {{< admonition type="note" >}} diff --git a/docs/sources/alerting/configure-notifications/template-notifications/reference.md b/docs/sources/alerting/configure-notifications/template-notifications/reference.md index 3bb4fe3f216..f0b84f67d16 100644 --- a/docs/sources/alerting/configure-notifications/template-notifications/reference.md +++ b/docs/sources/alerting/configure-notifications/template-notifications/reference.md @@ -23,20 +23,20 @@ weight: 400 ### Alert -| Name | Kind | Description | Example | -| ------------ | -------- | ------------------------------------------------------------------------------------ | --------------------- | -| Status | `string` | Firing or resolved | `{{ .Status }}` | -| Labels | `KV` | The labels for this alert | `{{ .Labels }}` | -| Annotations | `KV` | The annotations for this alert | `{{ .Annotations }}` | -| Values | `KV` | The values of all expressions, including Classic Conditions | `{{ .Values }}` | -| StartsAt | `Time` | The time the alert fired | `{{ .StartsAt }}` | -| EndsAt | `Time` | | `{{ .EndsAt }}` | -| GeneratorURL | `string` | A link to Grafana, or the Alertmanager if using an external Alertmanager | `{{ .GeneratorURL }}` | -| SilenceURL | `string` | A link to silence the alert | `{{ .SilenceURL }}` | -| DashboardURL | `string` | A link to the Grafana Dashboard if the alert has a Dashboard UID annotation | `{{ .DashboardURL }}` | -| PanelURL | `string` | A link to the panel if the alert has a Panel ID annotation | `{{ .PanelURL }}` | -| Fingerprint | `string` | A unique string that identifies the alert | `{{ .Fingerprint }}` | -| ValueString | `string` | A string that contains the labels and value of each reduced expression in the alert. | `{{ .ValueString }}` | +| Name | Kind | Description | Example | +| ------------ | -------- | ----------------------------------------------------------------------------------- | --------------------- | +| Status | `string` | Firing or resolved | `{{ .Status }}` | +| Labels | `KV` | The labels for this alert | `{{ .Labels }}` | +| Annotations | `KV` | The annotations for this alert | `{{ .Annotations }}` | +| Values | `KV` | The values of all expressions, including Classic Conditions | `{{ .Values }}` | +| StartsAt | `Time` | The time the alert fired | `{{ .StartsAt }}` | +| EndsAt | `Time` | | `{{ .EndsAt }}` | +| GeneratorURL | `string` | A link to Grafana, or the source of the alert if using an external alert generator | `{{ .GeneratorURL }}` | +| SilenceURL | `string` | A link to silence the alert | `{{ .SilenceURL }}` | +| DashboardURL | `string` | A link to the Grafana Dashboard if the alert has a Dashboard UID annotation | `{{ .DashboardURL }}` | +| PanelURL | `string` | A link to the panel if the alert has a Panel ID annotation | `{{ .PanelURL }}` | +| Fingerprint | `string` | A unique string that identifies the alert | `{{ .Fingerprint }}` | +| ValueString | `string` | A string that contains the labels and value of each reduced expression in the alert | `{{ .ValueString }}` | ### ExtendedData diff --git a/docs/sources/alerting/set-up/configure-rbac/_index.md b/docs/sources/alerting/set-up/configure-rbac/_index.md index 567b9dd9fab..6a15d86e724 100644 --- a/docs/sources/alerting/set-up/configure-rbac/_index.md +++ b/docs/sources/alerting/set-up/configure-rbac/_index.md @@ -52,4 +52,32 @@ Grafana Alerting has the following permissions. | `alert.provisioning:write` | n/a | Update all Grafana alert rules, notification policies, etc via provisioning API. Permissions to folders and data source are not required. | | `alert.provisioning.provenance:write` | n/a | Set provisioning status for alerting resources. Cannot be used alone. Requires user to have permissions to access resources | +Contact point permissions. To enable these permissions, enable the `alertingApiServer` feature toggle. + +| Action | Applicable scope | Description | +| -------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `alert.notifications.receivers:read` | `receivers:*`
`receivers:uid:*` | Read contact points. | +| `alert.notifications.receivers.secrets:read` | `receivers:*`
`receivers:uid:*` | Export contact points with decrypted secrets. | +| `alert.notifications.receivers:create` | n/a | Create a new contact points. The creator is automatically granted full access to the created contact point. | +| `alert.notifications.receivers:write` | `receivers:*`
`receivers:uid:*` | Update existing contact points. | +| `alert.notifications.receivers:delete` | `receivers:*`
`receivers:uid:*` | Update and delete existing contact points. | +| `receivers.permissions:read` | `receivers:*`
`receivers:uid:*` | Read permissions for contact points. | +| `receivers.permissions:write` | `receivers:*`
`receivers:uid:*` | Manage permissions for contact points. | + +Mute time interval permissions. To enable these permissions, enable the `alertingApiServer` feature toggle. + +| Action | Applicable scope | Description | +| ------------------------------------------- | ---------------- | -------------------------------------------------- | +| `alert.notifications.time-intervals:read` | n/a | Read mute time intervals. | +| `alert.notifications.time-intervals:write` | n/a | Create new or update existing mute time intervals. | +| `alert.notifications.time-intervals:delete` | n/a | Delete existing time intervals. | + +Notification template permissions. To enable these permissions, enable the `alertingApiServer` feature toggle. + +| Action | Applicable scope | Description | +| -------------------------------------- | ---------------- | ---------------------------------------- | +| `alert.notifications.templates:read` | n/a | Read templates. | +| `alert.notifications.templates:write` | n/a | Create new or update existing templates. | +| `alert.notifications.templates:delete` | n/a | Delete existing templates. | + To help plan your RBAC rollout strategy, refer to [Plan your RBAC rollout strategy](https://grafana.com/docs/grafana/next/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/). diff --git a/docs/sources/alerting/set-up/configure-rbac/access-roles/index.md b/docs/sources/alerting/set-up/configure-rbac/access-roles/index.md index 0b06882eb3d..2c9f3efa4db 100644 --- a/docs/sources/alerting/set-up/configure-rbac/access-roles/index.md +++ b/docs/sources/alerting/set-up/configure-rbac/access-roles/index.md @@ -57,12 +57,28 @@ Details of the fixed roles and the access they provide for Grafana Alerting are | Access to alert rules provisioning API: `fixed:alerting.provisioning:writer` | `alert.provisioning:read` and `alert.provisioning:write` | Manage all alert rules, notification policies, contact points, templates, in the organization using the provisioning API. | | Set provisioning status: `fixed:alerting.provisioning.status:writer` | `alert.provisioning.provenance:write` | Set provisioning rules for Alerting resources. Should be used together with other regular roles (Notifications Writer and/or Rules Writer.) | +If you have enabled the `alertingApiServer` feature toggle, an additional set of fixed roles is available. + +| Display name in UI / Fixed role | Permissions | Description | +| ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Contact Point Reader: `fixed:alerting.receivers:reader` | `alert.notifications.receivers:read` for scope `receivers:*` | Read all contact points. | +| Contact Point Creator: `fixed:alerting.receivers:creator` | `alert.notifications.receivers:create` | Create a new contact point. The user is automatically granted full access to the created contact point. | +| Contact Point Writer: `fixed:alerting.receivers:writer` | `alert.notifications.receivers:read`, `alert.notifications.receivers:write`, `alert.notifications.receivers:delete` for scope `receivers:*` and
`alert.notifications.receivers:create` | Create a new contact point and manage all existing contact points. | +| Templates Reader: `fixed:alerting.templates:reader` | `alert.notifications.templates:read` | Read all notification templates. | +| Templates Writer: `fixed:alerting.templates:writer` | `alert.notifications.templates:read`, `alert.notifications.templates:write`, `alert.notifications.templates:delete` | Create new and manage existing notification templates. | +| Time Intervals Reader: `fixed:alerting.time-intervals:reader` | `alert.notifications.time-intervals:read` | Read all time intervals. | +| Time Intervals Writer: `fixed:alerting.time-intervals:writer` | `alert.notifications.time-intervals:read`, `alert.notifications.time-intervals:write`, `alert.notifications.time-intervals:delete` | Create new and manage existing time intervals. | + ## Create custom roles Create custom roles of your own to manage permissions. Custom roles contain unique combinations of permissions, actions and scopes. Create a custom role when basic roles and fixed roles do not meet your permissions requirements. For more information on creating custom roles, refer to [Create custom roles](https://grafana.com/docs/grafana/latest/administration/roles-and-permissions/access-control/manage-rbac-roles/#create-custom-roles). +{{< admonition type="note" >}} +It is not recommended to create custom roles that include `alerting.notifications.receiver` actions with a scope other than `receivers:*`. The UID used in the scope is not stable and changes whenever a contact point is renamed. +{{< /admonition >}} + ### Examples The following examples give you an idea of how you can combine permissions for Grafana Alerting. diff --git a/docs/sources/alerting/set-up/configure-roles/index.md b/docs/sources/alerting/set-up/configure-roles/index.md index 6acb0734bb2..2d4470018de 100644 --- a/docs/sources/alerting/set-up/configure-roles/index.md +++ b/docs/sources/alerting/set-up/configure-roles/index.md @@ -64,3 +64,30 @@ To manage folder permissions, complete the following steps. 1. Hover your mouse cursor over a folder and click **Go to folder**. 1. Click **Manage permissions** from the Folder actions menu. 1. Update or add permissions as required. + +## Manage access using contact point permissions + +### Before you begin + +- Enable the `alertingApiServer` feature toggle. + +Extend or limit the access provided by a role to contact points by assigning permissions to individual contact point. + +This allows different users, teams, or service accounts to have customized access to read or modify specific contact points. + +Refer to the following table for details on the additional access provided by contact point permissions. + +| Folder permission | Additional Access | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| View | View and export contact point as well as select it on the Alert rule edit page | +| Edit | Update or delete the contact point | +| Admin | Same additional access as Edit and manage permissions for the contact point. User should have additional permissions to read users and teams. | + +### Steps + +To contact point permissions, complete the following steps. + +1. In the left-side menu, click **Contact points**. +1. Hover your mouse cursor over a contact point and click **More**. +1. Click **Manage permissions** from the actions menu. +1. Update or add permissions as required. diff --git a/docs/sources/datasources/prometheus/_index.md b/docs/sources/datasources/prometheus/_index.md index e5fa61ef1d6..433e14aedf8 100644 --- a/docs/sources/datasources/prometheus/_index.md +++ b/docs/sources/datasources/prometheus/_index.md @@ -155,7 +155,7 @@ We also bundle a dashboard within Grafana so you can start viewing your metrics 1. Navigate to the data source's [configuration page](ref:configure-prometheus-data-source). 1. Select the **Dashboards** tab. -This displays dashboards for Grafana and Prometheus. + This displays dashboards for Grafana and Prometheus. 1. Select **Import** for the dashboard to import. diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 46c13858bb3..a2d48a065ef 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -165,6 +165,7 @@ Experimental features might be changed or removed without prior notice. | `disableClassicHTTPHistogram` | Disables classic HTTP Histogram (use with enableNativeHTTPHistogram) | | `kubernetesSnapshots` | Routes snapshot requests from /api to the /apis endpoint | | `kubernetesDashboards` | Use the kubernetes API in the frontend for dashboards | +| `kubernetesDashboardsAPI` | Use the kubernetes API in the backend for dashboards | | `kubernetesFolders` | Use the kubernetes API in the frontend for folders, and route /api/folders requests to k8s | | `grafanaAPIServerTestingWithExperimentalAPIs` | Facilitate integration testing of experimental APIs | | `datasourceQueryTypes` | Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus) | diff --git a/docs/sources/shared/datasources/tempo-search-traceql.md b/docs/sources/shared/datasources/tempo-search-traceql.md index e51a25860f5..d1e1e3f7ffb 100644 --- a/docs/sources/shared/datasources/tempo-search-traceql.md +++ b/docs/sources/shared/datasources/tempo-search-traceql.md @@ -105,8 +105,8 @@ To add a tag, follow these steps: ### Optional: Use Aggregate by {{% admonition type="warning" %}} -**Aggregate by** is an [experimental feature](/docs/release-life-cycle/). Engineering and on-call support is not available. Documentation is either limited or not provided outside of code comments. No SLA is provided. -[Enable the `metricsSummary` feature toggle](/docs/grafana/latest/setup-grafana/configure-grafana/feature-toggles/) in Grafana to use this feature. Your Grafana Tempo data source must also point to a Tempo database with the [Metrics Summary API](https://grafana.com/docs/tempo/latest/api_docs/metrics-summary/) enabled. Contact Grafana Support to enable this feature in Grafana Cloud. +Metrics summary API and the **Aggregate by** feature are deprecated in Grafana Cloud and Grafana 11.3 and later. +It will be removed in a future release. {{% /admonition %}} Using **Aggregate by**, you can calculate RED metrics (total span count, percent erroring spans, and latency information) for spans of `kind=server` that match your filter criteria, grouped by one or more attributes. @@ -118,7 +118,16 @@ For additional information, refer to [Traces to metrics: Ad-hoc RED metrics in G {{< youtube id="xOolCpm2F8c" >}} -When you use **Aggregate by**, the selections you make determine how the information is reported in the Table. Every combination that matches selections in your data is listed in the table. +**Aggregate by** is an [experimental feature](/docs/release-life-cycle/) that is disabled by default. +[Enable the `metricsSummary` feature toggle](/docs/grafana/latest/setup-grafana/configure-grafana/feature-toggles/) in Grafana to use this feature. + +Your Grafana Tempo data source must also point to a Tempo database with the [Metrics Summary API](https://grafana.com/docs/tempo/latest/api_docs/metrics-summary/) enabled. +Contact Grafana Support to enable this feature in Grafana Cloud. + +#### Use Aggregate by + +When you use **Aggregate by**, the selections you make determine how the information is reported in the Table. +Every combination that matches selections in your data is listed in the table. Each aggregate value, for example `intrinsic`:`name`, has a corresponding column in the results table. For example, **names** matching `GET /:endpoint` with a **span.http.user_agent** of `k6/0.46` appeared in 31,466 spans. Instead of being listed by traces and associated spans, the query results are grouped by the selections in **Aggregate by**. diff --git a/go.mod b/go.mod index 5ab6b92c69d..34bbd6932f5 100644 --- a/go.mod +++ b/go.mod @@ -73,8 +73,8 @@ require ( github.com/gorilla/mux v1.8.1 // @grafana/grafana-backend-group github.com/gorilla/websocket v1.5.0 // @grafana/grafana-app-platform-squad github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724 // @grafana/alerting-backend - github.com/grafana/authlib v0.0.0-20240919120951-58259833c564 // @grafana/identity-access-team - github.com/grafana/authlib/claims v0.0.0-20240827210201-19d5347dd8dd // @grafana/identity-access-team + github.com/grafana/authlib v0.0.0-20241014135010-3e1f37f75699 // @grafana/identity-access-team + github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0 // @grafana/identity-access-team github.com/grafana/codejen v0.0.3 // @grafana/dataviz-squad github.com/grafana/cuetsy v0.1.11 // @grafana/grafana-as-code github.com/grafana/dataplane/examples v0.0.1 // @grafana/observability-metrics @@ -87,7 +87,7 @@ require ( github.com/grafana/grafana-cloud-migration-snapshot v1.3.0 // @grafana/grafana-operator-experience-squad github.com/grafana/grafana-google-sdk-go v0.1.0 // @grafana/partner-datasources github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79 // @grafana/grafana-backend-group - github.com/grafana/grafana-plugin-sdk-go v0.251.0 // @grafana/plugins-platform-backend + github.com/grafana/grafana-plugin-sdk-go v0.253.0 // @grafana/plugins-platform-backend github.com/grafana/grafana/pkg/aggregator v0.0.0-20240813192817-1b0e6b5c09b2 // @grafana/grafana-app-platform-squad github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240821155123-6891eb1d35da // @grafana/grafana-app-platform-squad github.com/grafana/grafana/pkg/apiserver v0.0.0-20240821155123-6891eb1d35da // @grafana/grafana-app-platform-squad diff --git a/go.sum b/go.sum index f707fdc6534..4bb578837b3 100644 --- a/go.sum +++ b/go.sum @@ -2249,10 +2249,10 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724 h1:u+ZM5TLkdeEoSWXgYWxc4XRfPHhXpR63MyHXJxbBLrc= github.com/grafana/alerting v0.0.0-20241010165806-807ddf183724/go.mod h1:QsnoKX/iYZxA4Cv+H+wC7uxutBD8qi8ZW5UJvD2TYmU= -github.com/grafana/authlib v0.0.0-20240919120951-58259833c564 h1:zYF/RBulpvMqPYR3gbzJZ8t/j/Eymn5FNidSYkueNCA= -github.com/grafana/authlib v0.0.0-20240919120951-58259833c564/go.mod h1:PFzXbCrn0GIpN4KwT6NP1l5Z1CPLfmKHnYx8rZzQcyY= -github.com/grafana/authlib/claims v0.0.0-20240827210201-19d5347dd8dd h1:sIlR7n38/MnZvX2qxDEszywXdI5soCwQ78aTDSARvus= -github.com/grafana/authlib/claims v0.0.0-20240827210201-19d5347dd8dd/go.mod h1:r+F8H6awwjNQt/KPZ2GNwjk8TvsJ7/gxzkXN26GlL/A= +github.com/grafana/authlib v0.0.0-20241014135010-3e1f37f75699 h1:+xSpRpQPhMXAE9z68u0zMzzIa78jy1UqFb4tMJczFNc= +github.com/grafana/authlib v0.0.0-20241014135010-3e1f37f75699/go.mod h1:fhuI+ulquEIVcLsbwPml9JapWQzg8EYBp29HteO62DM= +github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0 h1:XT/WvQCWVVOvXRJy0SCQHkhxXFHNRJ3+jzhW5PutEk8= +github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0/go.mod h1:r+F8H6awwjNQt/KPZ2GNwjk8TvsJ7/gxzkXN26GlL/A= github.com/grafana/codejen v0.0.3 h1:tAWxoTUuhgmEqxJPOLtJoxlPBbMULFwKFOcRsPRPXDw= github.com/grafana/codejen v0.0.3/go.mod h1:zmwwM/DRyQB7pfuBjTWII3CWtxcXh8LTwAYGfDfpR6s= github.com/grafana/cue v0.0.0-20230926092038-971951014e3f h1:TmYAMnqg3d5KYEAaT6PtTguL2GjLfvr6wnAX8Azw6tQ= @@ -2284,8 +2284,8 @@ github.com/grafana/grafana-google-sdk-go v0.1.0/go.mod h1:Vo2TKWfDVmNTELBUM+3lkr github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79 h1:r+mU5bGMzcXCRVAuOrTn54S80qbfVkvTdUJZfSfTNbs= github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79/go.mod h1:wc6Hbh3K2TgCUSfBC/BOzabItujtHMESZeFk5ZhdxhQ= github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk= -github.com/grafana/grafana-plugin-sdk-go v0.251.0 h1:gnOtxrC/1rqFvpSbQYyoZqkr47oWDlz4Q2L6Ozmsi3w= -github.com/grafana/grafana-plugin-sdk-go v0.251.0/go.mod h1:gCGN9kHY3KeX4qyni3+Kead38Q+85pYOrsDcxZp6AIk= +github.com/grafana/grafana-plugin-sdk-go v0.253.0 h1:KaCrqqsDgVIoT8hwvwuUMKV7QbHVlvRoFN5+U2rOXR8= +github.com/grafana/grafana-plugin-sdk-go v0.253.0/go.mod h1:gCGN9kHY3KeX4qyni3+Kead38Q+85pYOrsDcxZp6AIk= github.com/grafana/grafana/apps/playlist v0.0.0-20240917082838-e2bce38a7990 h1:uQMZE/z+Y+o/U0z/g8ckAHss7U7LswedilByA2535DU= github.com/grafana/grafana/apps/playlist v0.0.0-20240917082838-e2bce38a7990/go.mod h1:3Vi0xv/4OBkBw4R9GAERkSrBnx06qrjpmNBRisucuSM= github.com/grafana/grafana/pkg/aggregator v0.0.0-20240813192817-1b0e6b5c09b2 h1:2H9x4q53pkfUGtSNYD1qSBpNnxrFgylof/TYADb5xMI= diff --git a/go.work.sum b/go.work.sum index eb78119b883..44d2a672ed8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,7 +1,6 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20230802163732-1c33ebd9ecfa.1 h1:tdpHgTbmbvEIARu+bixzmleMi14+3imnpoFXz+Qzjp4= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20230802163732-1c33ebd9ecfa.1/go.mod h1:xafc+XIsTxTy76GJQ1TKgvJWsSugFBqMaN27WhUblew= cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= -cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cel.dev/expr v0.16.0 h1:yloc84fytn4zmJX2GU3TkXGsaieaV7dQ057Qs4sIG2Y= cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go/accessapproval v1.7.11 h1:MgtE8CI+YJWPGGHnxQ9z1VQqV87h+vSGy2MeM/m0ggQ= @@ -516,6 +515,8 @@ github.com/elastic/go-sysinfo v1.11.2/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/ettle/strcase v0.1.1 h1:htFueZyVeE1XNnMEfbqp5r67qAN/4r6ya1ysq8Q+Zcw= github.com/expr-lang/expr v1.16.2 h1:JvMnzUs3LeVHBvGFcXYmXo+Q6DPDmzrlcSBO6Wy3w4s= github.com/expr-lang/expr v1.16.2/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= @@ -598,7 +599,6 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 h1:EcQR3gusLHN46TAD+G+EbaaqJArt5vHhNpXAa12PQf4= @@ -906,7 +906,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad h1:fiWzISvDn0Csy5H0iwgAuJGQTUpVfEMJJd4nRFXogbc= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stoewer/parquet-cli v0.0.7 h1:rhdZODIbyMS3twr4OM3am8BPPT5pbfMcHLH93whDM5o= github.com/stoewer/parquet-cli v0.0.7/go.mod h1:bskxHdj8q3H1EmfuCqjViFoeO3NEvs5lzZAQvI8Nfjk= github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= @@ -1063,6 +1062,7 @@ go.opentelemetry.io/collector/service v0.95.0 h1:t6RUHV7ByFjkjPKGz5n6n4wIoXZLC8H go.opentelemetry.io/collector/service v0.95.0/go.mod h1:4yappQmDE5UZmLE9wwtj6IPM4W5KGLIYfObEAaejtQc= go.opentelemetry.io/contrib/config v0.4.0 h1:Xb+ncYOqseLroMuBesGNRgVQolXcXOhMj7EhGwJCdHs= go.opentelemetry.io/contrib/config v0.4.0/go.mod h1:drNk2xRqLWW4/amk6Uh1S+sDAJTc7bcEEN1GfJzj418= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0/go.mod h1:ImRBLMJv177/pwiLZ7tU7HDGNdBv7rS0HQ99eN/zBl8= go.opentelemetry.io/contrib/propagators/b3 v1.23.0 h1:aaIGWc5JdfRGpCafLRxMJbD65MfTa206AwSKkvGS0Hg= go.opentelemetry.io/contrib/propagators/b3 v1.23.0/go.mod h1:Gyz7V7XghvwTq+mIhLFlTgcc03UDroOg8vezs4NLhwU= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= @@ -1074,7 +1074,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.23.1 h1:ZqR go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.23.1/go.mod h1:D7ynngPWlGJrqyGSDOdscuv7uqttfCE3jcBvffDv9y4= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.23.1 h1:q/Nj5/2TZRIt6PderQ9oU0M00fzoe8UZuINGw6ETGTw= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.23.1/go.mod h1:DTE9yAu6r08jU3xa68GiSeI7oRcSEQ2RpKbbQGO+dWM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= @@ -1085,7 +1084,6 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 h1:s0PHtIkN+3xrbDO go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0/go.mod h1:hZlFbDbRt++MMPCCfSJfmhkGIWnX1h3XjkfxZUjLrIA= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/sdk/metric v1.26.0 h1:cWSks5tfriHPdWFnl+qpX3P681aAYqlZHcAyHw5aU9Y= go.opentelemetry.io/otel/sdk/metric v1.26.0/go.mod h1:ClMFFknnThJCksebJwz7KIyEDHO+nTB6gK8obLy8RyE= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= @@ -1118,11 +1116,9 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1134,8 +1130,6 @@ golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1150,15 +1144,12 @@ gonum.org/v1/plot v0.14.0 h1:+LBDVFYwFe4LHhdP8coW6296MBEY4nQ+Y4vuUpJopcE= gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:mCr1K1c8kX+1iSBREvU3Juo11CB+QOEWxbRS01wWl5M= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:OFMYQFHJ4TM3JRlWDZhJbZfra2uqc3WLBZiaaqP4DtU= google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf h1:T4tsZBlZYXK3j40sQNP5MBO32I+rn6ypV1PpklsiV8k= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= diff --git a/nx.json b/nx.json index 3ec7fc7711f..5e92cfe6851 100644 --- a/nx.json +++ b/nx.json @@ -3,7 +3,7 @@ "default": { "runner": "nx/tasks-runners/default", "options": { - "cacheableOperations": ["build"] + "cacheableOperations": ["build", "generate"] } } }, diff --git a/package.json b/package.json index 7efcf198bc5..40fbb729cfa 100644 --- a/package.json +++ b/package.json @@ -31,11 +31,11 @@ "lint:sass": "yarn stylelint '{public/sass,packages}/**/*.scss' --cache", "test:ci": "mkdir -p reports/junit && JEST_JUNIT_OUTPUT_DIR=reports/junit jest --ci --reporters=default --reporters=jest-junit -w ${TEST_MAX_WORKERS:-100%}", "lint:fix": "yarn lint:ts --fix", - "packages:build": "nx run-many -t build --projects='@grafana/*'", - "packages:clean": "rimraf ./npm-artifacts && lerna run clean --parallel", + "packages:build": "nx run-many -t build --projects='tag:scope:package'", + "packages:clean": "rimraf ./npm-artifacts && nx run-many -t clean --projects='tag:scope:package' --maxParallel=100", "packages:prepare": "lerna version --no-push --no-git-tag-version --force-publish --exact", "packages:pack": "mkdir -p ./npm-artifacts && lerna exec --no-private -- yarn pack --out \"../../npm-artifacts/%s-%v.tgz\"", - "packages:typecheck": "nx run-many -t typecheck --projects='@grafana/*'", + "packages:typecheck": "nx run-many -t typecheck --projects='tag:scope:package'", "prettier:check": "prettier --check --list-different=false --log-level=warn \"**/*.{ts,tsx,scss,md,mdx,json}\"", "prettier:checkDocs": "prettier --check --list-different=false --log-level=warn \"docs/**/*.md\" \"*.md\" \"packages/**/*.{ts,tsx,scss,md,mdx,json}\"", "prettier:write": "prettier --list-different \"**/*.{js,ts,tsx,scss,md,mdx,json}\" --write", @@ -58,10 +58,10 @@ "betterer:stats": "ts-node --transpile-only --project ./scripts/cli/tsconfig.json ./scripts/cli/reportBettererStats.ts", "betterer:issues": "ts-node --transpile-only --project ./scripts/cli/tsconfig.json ./scripts/cli/generateBettererIssues.ts", "generate-icons-bundle-cache-file": "node ./scripts/generate-icon-bundle.js", - "plugin:build": "nx run-many -t build --projects='@grafana-plugins/*'", - "plugin:build:commit": "nx run-many -t build:commit --projects='@grafana-plugins/*'", - "plugin:build:dev": "nx run-many -t dev --projects='@grafana-plugins/*' --maxParallel=100", - "generate-icons": "yarn workspace @grafana/saga-icons generate", + "plugin:build": "nx run-many -t build --projects='tag:scope:plugin'", + "plugin:build:commit": "nx run-many -t build:commit --projects='tag:scope:plugin'", + "plugin:build:dev": "nx run-many -t dev --projects='tag:scope:plugin' --maxParallel=100", + "generate-icons": "nx run grafana-icons:generate", "generate-apis": "rtk-query-codegen-openapi ./scripts/generate-rtk-apis.ts" }, "grafana": { diff --git a/packages/grafana-data/project.json b/packages/grafana-data/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-data/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts index 15dfed729c8..a41e61f556f 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts @@ -486,4 +486,52 @@ describe('fieldToTimeField', () => { ], }); }); + + it('should properly parse Unix timestamps - in seconds', () => { + const numberTimeField: Field = { + config: {}, + name: 'Unix second timestamps', + type: FieldType.number, + values: [1728397800, 1728397815, 1728397830], + }; + + expect(fieldToTimeField(numberTimeField, 'X')).toEqual({ + config: {}, + name: 'Unix second timestamps', + type: FieldType.time, + values: [1728397800000, 1728397815000, 1728397830000], + }); + }); + + it('should properly parse Unix timestamps - in millseconds (with format)', () => { + const numberTimeField: Field = { + config: {}, + name: 'Unix MS timestamps', + type: FieldType.number, + values: [1728397800000, 1728397815000, 1728397830000], + }; + + expect(fieldToTimeField(numberTimeField, 'x')).toEqual({ + config: {}, + name: 'Unix MS timestamps', + type: FieldType.time, + values: [1728397800000, 1728397815000, 1728397830000], + }); + }); + + it('should properly parse Unix timestamps - in millseconds (without format)', () => { + const numberTimeField: Field = { + config: {}, + name: 'Unix MS timestamps', + type: FieldType.number, + values: [1728397800000, 1728397815000, 1728397830000], + }; + + expect(fieldToTimeField(numberTimeField)).toEqual({ + config: {}, + name: 'Unix MS timestamps', + type: FieldType.time, + values: [1728397800000, 1728397815000, 1728397830000], + }); + }); }); diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts index 7eda28e04f5..3b3967a68b5 100644 --- a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts @@ -131,13 +131,17 @@ export function fieldToTimeField(field: Field, dateFormat?: string): Field { const timeValues = field.values.slice(); let firstDefined = timeValues.find((v) => v != null); - - let isISO8601 = typeof firstDefined === 'string' && iso8601Regex.test(firstDefined); + const convertToMS = typeof firstDefined === 'number' && dateFormat === 'X'; + const isISO8601 = typeof firstDefined === 'string' && iso8601Regex.test(firstDefined); for (let t = 0; t < timeValues.length; t++) { if (timeValues[t]) { let parsed = isISO8601 ? Date.parse(timeValues[t]) : dateTimeParse(timeValues[t], opts).valueOf(); - timeValues[t] = Number.isFinite(parsed) ? parsed : null; + if (Number.isFinite(parsed)) { + timeValues[t] = convertToMS ? parsed * 1000 : parsed; + } else { + timeValues[t] = null; + } } else { timeValues[t] = null; } @@ -254,10 +258,13 @@ function fieldToComplexField(field: Field): Field { */ export function ensureTimeField(field: Field, dateFormat?: string): Field { const firstValueTypeIsNumber = typeof field.values[0] === 'number'; + // if the format is unix seconds, we don't want to skip formatting + const isUnixSecondsFormat = dateFormat === 'X'; + if (field.type === FieldType.time && firstValueTypeIsNumber) { return field; //already time } - if (firstValueTypeIsNumber) { + if (firstValueTypeIsNumber && !isUnixSecondsFormat) { return { ...field, type: FieldType.time, //assumes it should be time diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 83738ec5cf1..964ccbc97f6 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -116,6 +116,7 @@ export interface FeatureToggles { kubernetesPlaylists?: boolean; kubernetesSnapshots?: boolean; kubernetesDashboards?: boolean; + kubernetesDashboardsAPI?: boolean; kubernetesFolders?: boolean; grafanaAPIServerTestingWithExperimentalAPIs?: boolean; datasourceQueryTypes?: boolean; diff --git a/packages/grafana-data/src/types/pluginExtensions.ts b/packages/grafana-data/src/types/pluginExtensions.ts index 54c1170681f..d9e53bb1a7b 100644 --- a/packages/grafana-data/src/types/pluginExtensions.ts +++ b/packages/grafana-data/src/types/pluginExtensions.ts @@ -223,7 +223,7 @@ type Dashboard = { // deprecated types -/** @deprecated - use PluginAddedComponentConfig instead */ +/** @deprecated - use PluginExtensionAddedLinkConfig instead */ export type PluginExtensionLinkConfig = { type: PluginExtensionTypes.link; title: string; diff --git a/packages/grafana-e2e-selectors/project.json b/packages/grafana-e2e-selectors/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-e2e-selectors/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-eslint-rules/project.json b/packages/grafana-eslint-rules/project.json new file mode 100644 index 00000000000..646c64168a8 --- /dev/null +++ b/packages/grafana-eslint-rules/project.json @@ -0,0 +1,6 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:build"], + "targets": {} +} diff --git a/packages/grafana-flamegraph/project.json b/packages/grafana-flamegraph/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-flamegraph/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-icons/project.json b/packages/grafana-icons/project.json new file mode 100644 index 00000000000..7950b1939c0 --- /dev/null +++ b/packages/grafana-icons/project.json @@ -0,0 +1,11 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "generate": { + "outputs": ["{projectRoot}/src/icons-gen"] + }, + "build": {} + } +} diff --git a/packages/grafana-o11y-ds-frontend/project.json b/packages/grafana-o11y-ds-frontend/project.json new file mode 100644 index 00000000000..2fe46a23eff --- /dev/null +++ b/packages/grafana-o11y-ds-frontend/project.json @@ -0,0 +1,6 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": {} +} diff --git a/packages/grafana-plugin-configs/project.json b/packages/grafana-plugin-configs/project.json new file mode 100644 index 00000000000..646c64168a8 --- /dev/null +++ b/packages/grafana-plugin-configs/project.json @@ -0,0 +1,6 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:build"], + "targets": {} +} diff --git a/packages/grafana-prometheus/project.json b/packages/grafana-prometheus/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-prometheus/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-runtime/project.json b/packages/grafana-runtime/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-runtime/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-schema/project.json b/packages/grafana-schema/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-schema/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-sql/project.json b/packages/grafana-sql/project.json new file mode 100644 index 00000000000..2fe46a23eff --- /dev/null +++ b/packages/grafana-sql/project.json @@ -0,0 +1,6 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": {} +} diff --git a/packages/grafana-ui/project.json b/packages/grafana-ui/project.json new file mode 100644 index 00000000000..9d13669dada --- /dev/null +++ b/packages/grafana-ui/project.json @@ -0,0 +1,8 @@ +{ + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:package", "type:ui"], + "targets": { + "build": {} + } +} diff --git a/packages/grafana-ui/src/components/Layout/Box/Box.tsx b/packages/grafana-ui/src/components/Layout/Box/Box.tsx index e8a8a6a0264..b42f40c1b75 100644 --- a/packages/grafana-ui/src/components/Layout/Box/Box.tsx +++ b/packages/grafana-ui/src/components/Layout/Box/Box.tsx @@ -1,4 +1,5 @@ import { css, cx } from '@emotion/css'; +import { Property } from 'csstype'; import { ElementType, forwardRef, PropsWithChildren } from 'react'; import * as React from 'react'; @@ -66,6 +67,7 @@ interface BoxProps extends FlexProps, SizeProps, Omit; /** Sets the HTML element that will be rendered as a Box. Defaults to 'div' */ element?: ElementType; + position?: ResponsiveProp; } export const Box = forwardRef>((props, ref) => { @@ -106,6 +108,7 @@ export const Box = forwardRef>((props, height, minHeight, maxHeight, + position, ...rest } = props; const styles = useStyles2( @@ -137,7 +140,8 @@ export const Box = forwardRef>((props, justifyContent, alignItems, boxShadow, - gap + gap, + position ); const sizeStyles = useStyles2(getSizeStyles, width, minWidth, maxWidth, height, minHeight, maxHeight); const Element = element ?? 'div'; @@ -204,7 +208,8 @@ const getStyles = ( justifyContent: BoxProps['justifyContent'], alignItems: BoxProps['alignItems'], boxShadow: BoxProps['boxShadow'], - gap: BoxProps['gap'] + gap: BoxProps['gap'], + position: BoxProps['position'] ) => { return { root: css([ @@ -299,6 +304,9 @@ const getStyles = ( getResponsiveStyle(theme, gap, (val) => ({ gap: theme.spacing(val), })), + getResponsiveStyle(theme, position, (val) => ({ + position: val, + })), ]), }; }; diff --git a/packages/grafana-ui/src/components/Table/Table.tsx b/packages/grafana-ui/src/components/Table/Table.tsx index 3beb04bda7c..bf7559f498c 100644 --- a/packages/grafana-ui/src/components/Table/Table.tsx +++ b/packages/grafana-ui/src/components/Table/Table.tsx @@ -44,6 +44,7 @@ export const Table = memo((props: Props) => { data, height, onCellFilterAdded, + onColumnResize, width, columnMinWidth = COLUMN_MIN_WIDTH, noHeader, @@ -128,7 +129,7 @@ export const Table = memo((props: Props) => { // Internal react table state reducer const stateReducer = useTableStateReducer({ - ...props, + onColumnResize, onSortByChange: (state) => { // Collapse all rows. This prevents a known bug that causes the size of the rows to be incorrect due to // using `VariableSizeList` and `useExpanded` together. @@ -138,6 +139,7 @@ export const Table = memo((props: Props) => { props.onSortByChange(state); } }, + data, }); const hasUniqueId = !!data.meta?.uniqueRowIdFields?.length; diff --git a/packages/grafana-ui/src/components/Table/reducer.ts b/packages/grafana-ui/src/components/Table/reducer.ts index 10e52e351e5..97ebf8869dd 100644 --- a/packages/grafana-ui/src/components/Table/reducer.ts +++ b/packages/grafana-ui/src/components/Table/reducer.ts @@ -2,14 +2,14 @@ import { useCallback } from 'react'; import { getFieldDisplayName } from '@grafana/data'; -import { TableSortByFieldState, GrafanaTableColumn, GrafanaTableState, Props } from './types'; +import { TableSortByFieldState, GrafanaTableColumn, GrafanaTableState, TableStateReducerProps, Props } from './types'; export interface ActionType { type: string; id: string | undefined; } -export function useTableStateReducer({ onColumnResize, onSortByChange, data }: Props) { +export function useTableStateReducer({ onColumnResize, onSortByChange, data }: TableStateReducerProps) { return useCallback( (newState: GrafanaTableState, action: ActionType) => { switch (action.type) { diff --git a/packages/grafana-ui/src/components/Table/types.ts b/packages/grafana-ui/src/components/Table/types.ts index 47801e3acdd..dda99074bd3 100644 --- a/packages/grafana-ui/src/components/Table/types.ts +++ b/packages/grafana-ui/src/components/Table/types.ts @@ -75,6 +75,12 @@ export interface GrafanaTableState extends TableState { export interface GrafanaTableRow extends Row, UseExpandedRowProps<{}> {} +export interface TableStateReducerProps { + onColumnResize?: TableColumnResizeActionCallback; + onSortByChange?: TableSortByActionCallback; + data: DataFrame; +} + export interface Props { ariaLabel?: string; data: DataFrame; diff --git a/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts b/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts index 2f57b80ef7a..82804c8254a 100644 --- a/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts +++ b/packages/grafana-ui/src/themes/_variables.dark.scss.tmpl.ts @@ -12,143 +12,66 @@ export const darkThemeVarsTemplate = (theme: GrafanaTheme2) => $theme-name: dark; -$colors-action-hover: ${theme.colors.action.hover}; -$colors-action-selected: ${theme.colors.action.selected}; - // New Colors // ------------------------- -$blue-light: ${theme.colors.primary.text}; $blue-base: ${theme.colors.primary.main}; -$blue-shade: ${theme.colors.primary.shade}; $red-base: ${theme.colors.error.main}; -$red-shade: ${theme.colors.error.shade}; $green-base: ${theme.colors.success.main}; -$green-shade: ${theme.colors.success.shade}; -$orange-dark: ${theme.v1.palette.orangeDark}; - -$gray98: ${theme.v1.palette.gray98}; -$gray95: ${theme.v1.palette.gray95}; -$gray85: ${theme.v1.palette.gray85}; -$gray70: ${theme.v1.palette.gray70}; -$gray60: ${theme.v1.palette.gray60}; -$gray33: ${theme.v1.palette.gray33}; -$gray25: ${theme.v1.palette.gray25}; -$gray15: ${theme.v1.palette.gray15}; -$gray10: ${theme.v1.palette.gray10}; -$gray05: ${theme.v1.palette.gray05}; // Grays // ------------------------- $black: ${theme.v1.palette.black}; $dark-1: ${theme.v1.palette.dark1}; -$dark-2: ${theme.v1.palette.dark2}; $dark-3: ${theme.v1.palette.dark3}; -$dark-4: ${theme.v1.palette.dark4}; -$dark-5: ${theme.v1.palette.dark5}; $dark-6: ${theme.v1.palette.dark6}; -$dark-7: ${theme.v1.palette.dark7}; -$dark-8: ${theme.v1.palette.dark8}; $dark-9: ${theme.v1.palette.dark9}; $dark-10: ${theme.v1.palette.dark10}; $gray-1: ${theme.v1.palette.gray1}; $gray-2: ${theme.v1.palette.gray2}; -$gray-3: ${theme.v1.palette.gray3}; -$gray-4: ${theme.v1.palette.gray4}; -$gray-5: ${theme.v1.palette.gray5}; $gray-6: ${theme.v1.palette.gray6}; $white: ${theme.v1.palette.white}; -$layer0: ${theme.colors.background.canvas}; -$layer1: ${theme.colors.background.primary}; $layer2: ${theme.colors.background.secondary}; -$divider: ${theme.colors.border.weak}; - -$border0: ${theme.colors.border.weak}; -$border1: ${theme.colors.border.medium}; - // Accent colors // ------------------------- $blue: ${theme.v1.palette.blue85}; $red: $red-base; $yellow: ${theme.v1.palette.yellow}; -$orange: ${theme.v1.palette.orange}; $purple: ${theme.v1.palette.purple}; -$variable: ${theme.colors.primary.text}; - -$brand-primary: ${theme.v1.palette.orange}; -$brand-success: ${theme.colors.success.main}; -$brand-warning: ${theme.colors.warning.main}; -$brand-danger: ${theme.colors.error.main}; - -$query-red: ${theme.colors.error.text}; -$query-green: ${theme.colors.success.text}; -$query-purple: #fe85fc; -$query-orange: ${theme.v1.palette.orange}; - -// Status colors -// -------------------------¨ -$online: ${theme.colors.success.text}; -$warn: ${theme.colors.warning.text}; -$critical: ${theme.colors.error.text}; // Scaffolding // ------------------------- $body-bg: ${theme.colors.background.canvas}; -$page-bg: ${theme.colors.background.canvas}; -$dashboard-bg: ${theme.colors.background.canvas}; -$text-color-strong: ${theme.colors.text.maxContrast}; $text-color: ${theme.colors.text.primary}; -$text-color-semi-weak: ${theme.colors.text.secondary}; $text-color-weak: ${theme.colors.text.secondary}; -$text-color-faint: ${theme.colors.text.disabled}; $text-color-emphasis: ${theme.colors.text.maxContrast}; -$text-blue: ${theme.colors.primary.text}; -$text-shadow-faint: 1px 1px 4px rgb(45, 45, 45); -$textShadow: none; - -// gradients -$brand-gradient-horizontal: ${theme.colors.gradients.brandHorizontal}; -$brand-gradient-vertical: ${theme.colors.gradients.brandVertical}; // Links // ------------------------- $link-color: ${theme.colors.text.primary}; $link-color-disabled: ${theme.colors.text.disabled}; $link-hover-color: ${theme.colors.text.maxContrast}; -$external-link-color: ${theme.colors.text.link}; // Typography // ------------------------- -$headings-color: ${theme.colors.text.primary}; -$abbr-border-color: $gray-2 !default; $text-muted: $text-color-weak; -$hr-border-color: $dark-9; - // Panel // ------------------------- $panel-bg: ${theme.components.panel.background}; -$panel-border: 1px solid ${theme.components.panel.borderColor}; -$panel-header-hover-bg: ${theme.colors.action.hover}; -$panel-box-shadow: ${theme.components.panel.boxShadow}; -$panel-corner: $panel-bg; // page header $page-header-bg: ${theme.colors.background.canvas}; $page-header-shadow: inset 0px -4px 14px $dark-3; $page-header-border-color: ${theme.colors.background.canvas}; -$divider-border-color: $gray-1; - // Graphite Target Editor $tight-form-func-bg: ${theme.colors.background.secondary}; -$tight-form-func-highlight-bg: ${theme.colors.emphasize(theme.colors.background.secondary, 0.03)}; -$modal-backdrop-bg: ${theme.colors.action.hover}; $code-tag-bg: $dark-1; $code-tag-border: $dark-9; @@ -159,76 +82,32 @@ $card-shadow: none; // Lists $list-item-bg: $card-background; -$list-item-hover-bg: $card-background-hover; -$list-item-shadow: $card-shadow; $empty-list-cta-bg: ${theme.colors.background.secondary}; // Scrollbars $scrollbarBackground: #404357; $scrollbarBackground2: $dark-10; -$scrollbarBorder: black; // Tables // ------------------------- $table-bg-accent: ${theme.colors.background.secondary}; -$table-border: ${theme.colors.border.medium}; -$table-bg-odd: ${theme.colors.emphasize(theme.colors.background.primary, 0.02)}; -$table-bg-hover: ${theme.colors.emphasize(theme.colors.background.primary, 0.05)}; // Buttons // ------------------------- -$btn-primary-bg: $blue-base; -$btn-primary-bg-hl: $blue-shade; - -$btn-secondary-bg: $dark-6; -$btn-secondary-bg-hl: color.adjust($dark-6, $lightness: 4%); - -$btn-success-bg: $green-base; -$btn-success-bg-hl: $green-shade; - -$btn-danger-bg: $red-base; -$btn-danger-bg-hl: $red-shade; - -$btn-inverse-bg: $dark-6; $btn-inverse-bg-hl: color.adjust($dark-6, $lightness: 4%); -$btn-inverse-text-color: $link-color; -$btn-inverse-text-shadow: 0px 1px 0 rgba(0, 0, 0, 0.1); - -$btn-link-color: $gray-3; - -$iconContainerBackground: $black; $btn-divider-left: $dark-9; $btn-divider-right: $dark-3; $btn-drag-image: '../img/grab_dark.svg'; -$navbar-btn-gicon-brightness: brightness(0.5); - -$btn-active-box-shadow: 0px 0px 4px rgba(255, 120, 10, 0.5); - // Forms // ------------------------- $input-bg: ${theme.components.input.background}; -$input-bg-disabled: ${theme.colors.action.disabledBackground}; $input-color: ${theme.components.input.text}; $input-border-color: ${theme.components.input.borderColor}; -$input-box-shadow: none; -$input-border-focus: ${theme.colors.primary.border}; -$input-box-shadow-focus: ${theme.colors.primary.border} !default; -$input-color-placeholder: ${theme.colors.text.disabled}; -$input-label-bg: ${theme.colors.background.secondary}; -$input-color-select-arrow: $white; - -// Search -$search-shadow: 0 0 30px 0 $black; - -// Typeahead -$typeahead-shadow: 0 5px 10px 0 $black; -$typeahead-selected-bg: $dark-9; -$typeahead-selected-color: $yellow; // Dropdowns // ------------------------- @@ -236,48 +115,20 @@ $dropdownBackground: ${theme.colors.background.primary}; $dropdownBorder: ${theme.colors.border.weak}; $dropdownDividerTop: ${theme.colors.border.weak}; $dropdownDividerBottom: ${theme.colors.border.weak}; -$dropdownShadow: ${theme.shadows.z3}; $dropdownLinkColor: $link-color; $dropdownLinkColorHover: $white; $dropdownLinkColorActive: $white; $dropdownLinkBackgroundHover: $dark-9; -// Horizontal forms & lists -// ------------------------- -$horizontalComponentOffset: 180px; - -// Navbar -// ------------------------- -$navbarHeight: 55px; -$navbarBorder: 1px solid $dark-6; - -// Sidemenu -// ------------------------- -$side-menu-bg: $panel-bg; -$side-menu-bg-mobile: $panel-bg; -$side-menu-border: none; -$side-menu-item-hover-bg: ${theme.colors.background.secondary}; -$side-menu-shadow: 0 0 30px #111; -$side-menu-icon-color: ${theme.v1.palette.gray70}; -$side-menu-header-color: ${theme.colors.text.primary}; - // Menu dropdowns // ------------------------- $menu-dropdown-bg: ${theme.colors.background.primary}; $menu-dropdown-hover-bg: ${theme.colors.action.hover}; $menu-dropdown-shadow: ${theme.shadows.z3}; -// Tabs -// ------------------------- -$tab-border-color: $dark-9; - // Form states and alerts // ------------------------- -$warning-text-color: ${theme.colors.warning.text}; -$error-text-color: ${theme.colors.error.text}; -$success-text-color: ${theme.colors.success.text}; - $alert-error-bg: ${theme.colors.error.main}; $alert-success-bg: ${theme.colors.success.main}; $alert-warning-bg: ${theme.colors.warning.main}; @@ -285,16 +136,12 @@ $alert-info-bg: ${theme.colors.warning.main}; // Tooltips and popovers // ------------------------- -$tooltipArrowWidth: 5px; $tooltipLinkColor: $link-color; -$tooltipExternalLinkColor: $external-link-color; +$tooltipExternalLinkColor: ${theme.colors.text.link}; $graph-tooltip-bg: $dark-1; $tooltipBackground: ${theme.components.tooltip.background}; $tooltipColor: ${theme.components.tooltip.text}; -$tooltipArrowColor: ${theme.components.tooltip.background}; -$tooltipBackgroundError: ${theme.colors.error.main}; -$tooltipShadow: ${theme.shadows.z2}; $popover-bg: ${theme.colors.background.primary}; $popover-color: ${theme.colors.text.primary}; @@ -304,10 +151,7 @@ $popover-shadow: ${theme.shadows.z3}; $popover-help-bg: $tooltipBackground; $popover-help-color: $text-color; -$popover-error-bg: $btn-danger-bg; - -$popover-code-bg: $popover-bg; -$popover-code-boxshadow: $tooltipShadow; +$popover-error-bg: $red-base; // images $checkboxImageUrl: '../img/checkbox.png'; @@ -315,46 +159,6 @@ $checkboxImageUrl: '../img/checkbox.png'; // info box $info-box-border-color: $blue-base; -// footer -$footer-link-color: $gray-2; -$footer-link-hover: $gray-4; - -// json-explorer -$json-explorer-default-color: $text-color; -$json-explorer-string-color: #23d662; -$json-explorer-number-color: $variable; -$json-explorer-boolean-color: $variable; -$json-explorer-null-color: #eec97d; -$json-explorer-undefined-color: rgb(239, 143, 190); -$json-explorer-function-color: #fd48cb; -$json-explorer-rotate-time: 100ms; -$json-explorer-toggler-opacity: 0.6; -$json-explorer-bracket-color: #9494ff; -$json-explorer-key-color: #23a0db; -$json-explorer-url-color: #027bff; - -// Changelog and diff -// ------------------------- -$diff-label-bg: ${theme.colors.action.hover}; -$diff-label-fg: $white; - -$diff-group-bg: ${theme.colors.background.secondary}; -$diff-arrow-color: $white; - -$diff-json-bg: ${theme.colors.background.secondary}; -$diff-json-fg: ${theme.colors.text.primary}; - -$diff-json-added: $blue-shade; -$diff-json-deleted: $red-shade; - -$diff-json-old: #a04338; -$diff-json-new: #457740; - -$diff-json-changed-fg: $gray-5; -$diff-json-changed-num: $text-color; - -$diff-json-icon: $gray-5; - //Switch Slider // ------------------------- $switch-bg: $input-bg; @@ -369,36 +173,4 @@ $checkbox-bg: $dark-1; $checkbox-border: 1px solid $gray-1; $checkbox-checked-bg: linear-gradient(0deg, #eb7b18, #d44a3a); $checkbox-color: $dark-1; - -//Panel Edit -// ------------------------- -$panel-editor-shadow: 0 0 20px black; -$panel-editor-side-menu-shadow: drop-shadow(0 0 10px $black); -$panel-editor-viz-item-shadow: 0 0 8px $dark-10; -$panel-editor-viz-item-border: 1px solid $dark-10; -$panel-editor-viz-item-shadow-hover: 0 0 4px $blue-light; -$panel-editor-viz-item-border-hover: 1px solid $blue-light; -$panel-editor-viz-item-bg: $input-bg; -$panel-editor-tabs-line-color: #e3e3e3; - -$panel-editor-viz-item-bg-hover: color.adjust($blue-base, $lightness: -46%); - -$panel-grid-placeholder-bg: color.adjust(${theme.v1.palette.blue77}, $lightness: -30%); -$panel-grid-placeholder-shadow: 0 0 4px ${theme.v1.palette.blue80}; - -// logs -$logs-color-unknown: $gray-2; - -// toggle-group -$button-toggle-group-btn-active-bg: linear-gradient(90deg, #eb7b18, #d44a3a); -$button-toggle-group-btn-active-shadow: inset 0 0 4px $black; -$button-toggle-group-btn-separator-border: 1px solid $dark-2; - -$vertical-resize-handle-bg: $dark-10; -$vertical-resize-handle-dots: $gray-1; -$vertical-resize-handle-dots-hover: $gray-2; - -// Calendar -$calendar-bg-days: $input-bg; -$calendar-bg-now: $dark-10; `; diff --git a/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts b/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts index 85d5ce94f54..a88d28df02c 100644 --- a/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts +++ b/packages/grafana-ui/src/themes/_variables.light.scss.tmpl.ts @@ -4,8 +4,6 @@ import { GrafanaTheme2 } from '@grafana/data'; import { renderGeneratedFileBanner } from '../utils/generatedFileBanner'; -import { styleMixins } from '.'; - export const lightThemeVarsTemplate = (theme: GrafanaTheme2) => `${renderGeneratedFileBanner('grafana-ui/src/themes/light.ts', 'grafana-ui/src/themes/_variable.light.scss.tmpl.ts')} @use 'sass:color'; @@ -14,42 +12,20 @@ export const lightThemeVarsTemplate = (theme: GrafanaTheme2) => $theme-name: light; -$colors-action-hover: ${theme.colors.action.hover}; -$colors-action-selected: ${theme.colors.action.selected}; - // New Colors // ------------------------- -$blue-light: ${theme.colors.primary.text}; $blue-base: ${theme.colors.primary.main}; -$blue-shade: ${theme.colors.primary.shade}; $red-base: ${theme.colors.error.main}; -$red-shade: ${theme.colors.error.shade}; $green-base: ${theme.colors.success.main}; -$green-shade: ${theme.colors.success.shade}; -$orange-dark: ${theme.v1.palette.orangeDark}; - -$gray98: ${theme.v1.palette.gray98}; -$gray95: ${theme.v1.palette.gray95}; -$gray85: ${theme.v1.palette.gray85}; -$gray70: ${theme.v1.palette.gray70}; -$gray60: ${theme.v1.palette.gray60}; -$gray33: ${theme.v1.palette.gray33}; -$gray25: ${theme.v1.palette.gray25}; -$gray15: ${theme.v1.palette.gray15}; -$gray10: ${theme.v1.palette.gray10}; -$gray05: ${theme.v1.palette.gray05}; // Grays // ------------------------- $black: ${theme.v1.palette.black}; -$dark-1: ${theme.v1.palette.dark1}; $dark-2: ${theme.v1.palette.dark2}; -$dark-4: ${theme.v1.palette.dark4}; $dark-10: ${theme.v1.palette.dark10}; $gray-1: ${theme.v1.palette.gray1}; $gray-2: ${theme.v1.palette.gray2}; -$gray-3: ${theme.v1.palette.gray3}; $gray-4: ${theme.v1.palette.gray4}; $gray-5: ${theme.v1.palette.gray5}; $gray-6: ${theme.v1.palette.gray6}; @@ -57,95 +33,45 @@ $gray-7: ${theme.v1.palette.gray7}; $white: ${theme.v1.palette.white}; -$layer0: ${theme.colors.background.canvas}; -$layer1: ${theme.colors.background.primary}; $layer2: ${theme.colors.background.secondary}; -$divider: ${theme.colors.border.weak}; -$border0: ${theme.colors.border.weak}; -$border1: ${theme.colors.border.medium}; - // Accent colors // ------------------------- $blue: ${theme.colors.primary.text}; $red: $red-base; $yellow: ${theme.v1.palette.yellow}; -$orange: ${theme.v1.palette.orange}; $purple: ${theme.v1.palette.purple}; -$variable: ${theme.colors.primary.text}; - -$brand-primary: ${theme.v1.palette.orange}; -$brand-success: ${theme.colors.success.main}; -$brand-warning: ${theme.colors.warning.main}; -$brand-danger: ${theme.colors.error.main}; - -$query-red: ${theme.colors.error.text}; -$query-green: ${theme.colors.success.text}; -$query-purple: #fe85fc; -$query-orange: ${theme.v1.palette.orange}; - -// Status colors -// ------------------------- -$online: ${theme.colors.success.text}; -$warn: ${theme.colors.warning.text}; -$critical: ${theme.colors.error.text}; - // Scaffolding // ------------------------- $body-bg: ${theme.colors.background.canvas}; -$page-bg: ${theme.colors.background.canvas}; -$dashboard-bg: ${theme.colors.background.canvas}; $text-color: ${theme.colors.text.primary}; -$text-color-strong: ${theme.colors.text.maxContrast}; -$text-color-semi-weak: ${theme.colors.text.secondary}; $text-color-weak: ${theme.colors.text.secondary}; -$text-color-faint: ${theme.colors.text.disabled}; $text-color-emphasis: ${theme.colors.text.maxContrast}; -$text-blue: ${theme.colors.primary.text}; - -$text-shadow-faint: none; - -// gradients -$brand-gradient-horizontal: ${theme.colors.gradients.brandHorizontal}; -$brand-gradient-vertical: ${theme.colors.gradients.brandVertical}; // Links // ------------------------- $link-color: ${theme.colors.text.primary}; $link-color-disabled: ${theme.colors.text.disabled}; $link-hover-color: ${theme.colors.text.maxContrast}; -$external-link-color: ${theme.colors.text.link}; // Typography // ------------------------- -$headings-color: ${theme.colors.text.primary}; -$abbr-border-color: $gray-2 !default; $text-muted: $text-color-weak; -$hr-border-color: $gray-4 !default; - // Panel // ------------------------- $panel-bg: ${theme.components.panel.background}; -$panel-border: 1px solid ${theme.components.panel.borderColor}; -$panel-header-hover-bg: ${theme.colors.action.hover}; -$panel-box-shadow: ${theme.components.panel.boxShadow}; -$panel-corner: $panel-bg; // Page header $page-header-bg: ${theme.colors.background.canvas}; $page-header-shadow: inset 0px -3px 10px $gray-6; $page-header-border-color: ${theme.colors.background.canvas}; -$divider-border-color: $gray-2; - // Graphite Target Editor $tight-form-func-bg: ${theme.colors.background.secondary}; -$tight-form-func-highlight-bg: ${styleMixins.hoverColor(theme.colors.background.secondary, theme)}; -$modal-backdrop-bg: ${theme.colors.background.primary}; $code-tag-bg: $gray-6; $code-tag-border: $gray-4; @@ -156,76 +82,32 @@ $card-shadow: none; // Lists $list-item-bg: $gray-7; -$list-item-hover-bg: $gray-6; -$list-item-shadow: $card-shadow; $empty-list-cta-bg: $gray-6; // Scrollbars $scrollbarBackground: $gray-4; $scrollbarBackground2: $gray-4; -$scrollbarBorder: $gray-7; // Tables // ------------------------- $table-bg-accent: ${theme.colors.background.secondary}; -$table-border: ${theme.colors.border.medium}; -$table-bg-odd: ${theme.colors.emphasize(theme.colors.background.primary, 0.02)}; -$table-bg-hover: ${theme.colors.emphasize(theme.colors.background.primary, 0.05)}; // Buttons // ------------------------- -$btn-secondary-bg: $gray-5; -$btn-secondary-bg-hl: $gray-4; - -$btn-primary-bg: $blue-base; -$btn-primary-bg-hl: $blue-shade; - -$btn-success-bg: $green-base; -$btn-success-bg-hl: $green-shade; - -$btn-danger-bg: $red-base; -$btn-danger-bg-hl: $red-shade; - -$btn-inverse-bg: $gray-5; $btn-inverse-bg-hl: $gray-4; -$btn-inverse-text-color: $gray-1; -$btn-inverse-text-shadow: 0 1px 0 rgba(255, 255, 255, 0.4); - -$btn-link-color: $gray-1; - -$iconContainerBackground: $white; $btn-divider-left: $gray-4; $btn-divider-right: $gray-7; $btn-drag-image: '../img/grab_light.svg'; -$navbar-btn-gicon-brightness: brightness(1.5); - -$btn-active-box-shadow: 0px 0px 4px rgba(234, 161, 51, 0.6); - // Forms // ------------------------- $input-bg: ${theme.components.input.background}; -$input-bg-disabled: ${theme.colors.action.disabledBackground}; $input-color: ${theme.components.input.text}; $input-border-color: ${theme.components.input.borderColor}; -$input-box-shadow: none; -$input-border-focus: ${theme.v1.palette.blue95}; -$input-box-shadow-focus: ${theme.v1.palette.blue95}; -$input-color-placeholder: ${theme.colors.text.disabled}; -$input-label-bg: ${theme.colors.background.secondary}; -$input-color-select-arrow: ${theme.v1.palette.gray60}; - -// search -$search-shadow: 0 1px 5px 0 $gray-5; - -// Typeahead -$typeahead-shadow: 0 5px 10px 0 $gray-5; -$typeahead-selected-bg: $gray-6; -$typeahead-selected-color: $yellow; // Dropdowns // ------------------------- @@ -233,7 +115,6 @@ $dropdownBackground: ${theme.colors.background.primary}; $dropdownBorder: ${theme.colors.border.weak}; $dropdownDividerTop: ${theme.colors.border.weak}; $dropdownDividerBottom: ${theme.colors.border.weak}; -$dropdownShadow: ${theme.shadows.z3}; $dropdownLinkColor: $dark-2; $dropdownLinkColorHover: $link-color; @@ -241,42 +122,14 @@ $dropdownLinkColorActive: $link-color; $dropdownLinkBackgroundHover: $gray-6; -// Horizontal forms & lists -// ------------------------- -$horizontalComponentOffset: 180px; - -// Navbar -// ------------------------- -$navbarHeight: 52px; -$navbarBorder: 1px solid $gray-5; - -// Sidemenu -// ------------------------- -$side-menu-bg: ${theme.v1.palette.gray15}; -$side-menu-border: 1px solid ${theme.v1.palette.gray25}; -$side-menu-bg-mobile: rgba(0, 0, 0, 0); //$gray-6; -$side-menu-item-hover-bg: ${theme.v1.palette.gray25}; -$side-menu-shadow: 5px 0px 10px -5px $gray-1; -$side-menu-link-color: $gray-4; -$side-menu-icon-color: ${theme.v1.palette.gray70}; -$side-menu-header-color: ${theme.v1.palette.gray95}; - // Menu dropdowns // ------------------------- $menu-dropdown-bg: ${theme.colors.background.primary}; $menu-dropdown-hover-bg: ${theme.colors.action.hover}; $menu-dropdown-shadow: ${theme.shadows.z3}; -// Tabs -// ------------------------- -$tab-border-color: $gray-5; - // Form states and alerts // ------------------------- -$warning-text-color: ${theme.colors.warning.text}; -$error-text-color: ${theme.colors.error.text}; -$success-text-color: ${theme.colors.success.text}; - $alert-error-bg: ${theme.colors.error.main}; $alert-success-bg: ${theme.colors.success.main}; $alert-warning-bg: ${theme.colors.warning.main}; @@ -285,9 +138,6 @@ $alert-info-bg: ${theme.colors.warning.main}; // Tooltips and popovers $tooltipBackground: ${theme.components.tooltip.background}; $tooltipColor: ${theme.components.tooltip.text}; -$tooltipArrowColor: ${theme.components.tooltip.background}; -$tooltipBackgroundError: ${theme.colors.error.main}; -$tooltipShadow: ${theme.shadows.z2}; $popover-bg: ${theme.colors.background.primary}; $popover-color: ${theme.colors.text.primary}; @@ -297,63 +147,19 @@ $popover-shadow: ${theme.shadows.z3}; $graph-tooltip-bg: $gray-5; -$tooltipArrowWidth: 5px; $tooltipLinkColor: color.adjust($tooltipColor, $lightness: 5%); $tooltipExternalLinkColor: #6E9FFF; -$popover-error-bg: $btn-danger-bg; +$popover-error-bg: $red-base; $popover-help-bg: $tooltipBackground; $popover-help-color: $tooltipColor; -$popover-code-bg: ${theme.colors.background.primary}; -$popover-code-boxshadow: 0 0 5px $gray60; - // images $checkboxImageUrl: '../img/checkbox_white.png'; // info box $info-box-border-color: $blue-base; -// footer -$footer-link-color: $gray-1; -$footer-link-hover: $dark-2; - -// json explorer -$json-explorer-default-color: black; -$json-explorer-string-color: green; -$json-explorer-number-color: $blue-base; -$json-explorer-boolean-color: $red-base; -$json-explorer-null-color: #855a00; -$json-explorer-undefined-color: rgb(202, 11, 105); -$json-explorer-function-color: #ff20ed; -$json-explorer-rotate-time: 100ms; -$json-explorer-toggler-opacity: 0.6; -$json-explorer-bracket-color: $blue-base; -$json-explorer-key-color: #00008b; -$json-explorer-url-color: $blue-base; - -// Changelog and diff -// ------------------------- -$diff-label-bg: ${theme.colors.action.hover}; -$diff-label-fg: $gray-2; - -$diff-arrow-color: $dark-2; -$diff-group-bg: ${theme.colors.background.secondary}; - -$diff-json-bg: ${theme.colors.background.secondary}; -$diff-json-fg: ${theme.colors.text.primary}; - -$diff-json-added: $blue-shade; -$diff-json-deleted: $red-shade; - -$diff-json-old: #5a372a; -$diff-json-new: #664e33; - -$diff-json-changed-fg: $gray-7; -$diff-json-changed-num: $gray-4; - -$diff-json-icon: $gray-4; - //Switch Slider // ------------------------- $switch-bg: $white; @@ -365,40 +171,7 @@ $switch-slider-shadow: 0 0 3px $dark-2; //Checkbox // ------------------------- $checkbox-bg: $gray-6; -$checkbox-border: 1px solid $gray-3; +$checkbox-border: 1px solid ${theme.v1.palette.gray3}; $checkbox-checked-bg: linear-gradient(0deg, #ff9830, #e55400); $checkbox-color: $gray-7; - -//Panel Edit -// ------------------------- -$panel-editor-shadow: 0px 0px 8px $gray-3; -$panel-editor-side-menu-shadow: drop-shadow(0 0 2px $gray-3); -$panel-editor-viz-item-shadow: 0 0 4px $gray-3; -$panel-editor-viz-item-border: 1px solid $gray-3; -$panel-editor-viz-item-shadow-hover: 0 0 4px $blue-light; -$panel-editor-viz-item-border-hover: 1px solid $blue-light; -$panel-editor-viz-item-bg: $card-background; -$panel-editor-tabs-line-color: $dark-2; - -$panel-editor-viz-item-bg-hover: color.adjust($blue-base, $lightness: 45%); - -$panel-grid-placeholder-bg: color.adjust(${theme.v1.palette.blue95}, $lightness: 30%); -$panel-grid-placeholder-shadow: 0 0 4px ${theme.v1.palette.blue95}; - -// logs -$logs-color-unknown: $gray-5; - -// toggle-group -$button-toggle-group-btn-active-bg: $brand-primary; -$button-toggle-group-btn-active-shadow: inset 0 0 4px $white; -$button-toggle-group-btn-separator-border: 1px solid $gray-6; - -$vertical-resize-handle-bg: $gray-4; -$vertical-resize-handle-dots: $gray-3; -$vertical-resize-handle-dots-hover: $gray-2; - -// Calendar -$calendar-bg-days: $white; -$calendar-bg-now: $gray-6; - `; diff --git a/pkg/aggregator/go.mod b/pkg/aggregator/go.mod index 601011db495..b62d1c5cb5d 100644 --- a/pkg/aggregator/go.mod +++ b/pkg/aggregator/go.mod @@ -4,7 +4,7 @@ go 1.23.1 require ( github.com/emicklei/go-restful/v3 v3.11.0 - github.com/grafana/grafana-plugin-sdk-go v0.251.0 + github.com/grafana/grafana-plugin-sdk-go v0.253.0 github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240808213237-f4d2e064f435 github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38 diff --git a/pkg/aggregator/go.sum b/pkg/aggregator/go.sum index 51002ade3a7..a113f3c027b 100644 --- a/pkg/aggregator/go.sum +++ b/pkg/aggregator/go.sum @@ -130,8 +130,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/grafana-plugin-sdk-go v0.251.0 h1:gnOtxrC/1rqFvpSbQYyoZqkr47oWDlz4Q2L6Ozmsi3w= -github.com/grafana/grafana-plugin-sdk-go v0.251.0/go.mod h1:gCGN9kHY3KeX4qyni3+Kead38Q+85pYOrsDcxZp6AIk= +github.com/grafana/grafana-plugin-sdk-go v0.253.0 h1:KaCrqqsDgVIoT8hwvwuUMKV7QbHVlvRoFN5+U2rOXR8= +github.com/grafana/grafana-plugin-sdk-go v0.253.0/go.mod h1:gCGN9kHY3KeX4qyni3+Kead38Q+85pYOrsDcxZp6AIk= github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240808213237-f4d2e064f435 h1:lmw60EW7JWlAEvgggktOyVkH4hF1m/+LSF/Ap0NCyi8= github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240808213237-f4d2e064f435/go.mod h1:ORVFiW/KNRY52lNjkGwnFWCxNVfE97bJG2jr2fetq0I= github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 h1:SNEeqY22DrGr5E9kGF1mKSqlOom14W9+b1u4XEGJowA= diff --git a/pkg/apimachinery/go.mod b/pkg/apimachinery/go.mod index f0ab30ee9c0..1ef6f1e82d9 100644 --- a/pkg/apimachinery/go.mod +++ b/pkg/apimachinery/go.mod @@ -3,8 +3,8 @@ module github.com/grafana/grafana/pkg/apimachinery go 1.23.1 require ( - github.com/grafana/authlib v0.0.0-20240919120951-58259833c564 // @grafana/identity-access-team - github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e // @grafana/identity-access-team + github.com/grafana/authlib v0.0.0-20241014135010-3e1f37f75699 // @grafana/identity-access-team + github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0 // @grafana/identity-access-team github.com/stretchr/testify v1.9.0 k8s.io/apimachinery v0.31.1 k8s.io/apiserver v0.31.1 diff --git a/pkg/apimachinery/go.sum b/pkg/apimachinery/go.sum index f2a1732dde0..61b987b71d6 100644 --- a/pkg/apimachinery/go.sum +++ b/pkg/apimachinery/go.sum @@ -28,10 +28,10 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/grafana/authlib v0.0.0-20240919120951-58259833c564 h1:zYF/RBulpvMqPYR3gbzJZ8t/j/Eymn5FNidSYkueNCA= -github.com/grafana/authlib v0.0.0-20240919120951-58259833c564/go.mod h1:PFzXbCrn0GIpN4KwT6NP1l5Z1CPLfmKHnYx8rZzQcyY= -github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e h1:ng5SopWamGS0MHaCj2e5huWYxAfMeCrj1l/dbJnfiow= -github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e/go.mod h1:r+F8H6awwjNQt/KPZ2GNwjk8TvsJ7/gxzkXN26GlL/A= +github.com/grafana/authlib v0.0.0-20241014135010-3e1f37f75699 h1:+xSpRpQPhMXAE9z68u0zMzzIa78jy1UqFb4tMJczFNc= +github.com/grafana/authlib v0.0.0-20241014135010-3e1f37f75699/go.mod h1:fhuI+ulquEIVcLsbwPml9JapWQzg8EYBp29HteO62DM= +github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0 h1:XT/WvQCWVVOvXRJy0SCQHkhxXFHNRJ3+jzhW5PutEk8= +github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0/go.mod h1:r+F8H6awwjNQt/KPZ2GNwjk8TvsJ7/gxzkXN26GlL/A= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= diff --git a/pkg/apiserver/go.mod b/pkg/apiserver/go.mod index 3a728b84ae9..78fb9e6826f 100644 --- a/pkg/apiserver/go.mod +++ b/pkg/apiserver/go.mod @@ -4,7 +4,7 @@ go 1.23.1 require ( github.com/google/go-cmp v0.6.0 - github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e + github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0 github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240701135906-559738ce6ae1 github.com/prometheus/client_golang v1.20.4 github.com/stretchr/testify v1.9.0 diff --git a/pkg/apiserver/go.sum b/pkg/apiserver/go.sum index 73455d71aea..113c88a38d5 100644 --- a/pkg/apiserver/go.sum +++ b/pkg/apiserver/go.sum @@ -78,8 +78,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e h1:ng5SopWamGS0MHaCj2e5huWYxAfMeCrj1l/dbJnfiow= -github.com/grafana/authlib/claims v0.0.0-20240903121118-16441568af1e/go.mod h1:r+F8H6awwjNQt/KPZ2GNwjk8TvsJ7/gxzkXN26GlL/A= +github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0 h1:XT/WvQCWVVOvXRJy0SCQHkhxXFHNRJ3+jzhW5PutEk8= +github.com/grafana/authlib/claims v0.0.0-20240926100702-4aee62663da0/go.mod h1:r+F8H6awwjNQt/KPZ2GNwjk8TvsJ7/gxzkXN26GlL/A= github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240701135906-559738ce6ae1 h1:ItDcDxUjVLPKja+hogpqgW/kj8LxUL2qscelXIsN1Bs= github.com/grafana/grafana/pkg/apimachinery v0.0.0-20240701135906-559738ce6ae1/go.mod h1:DkxMin+qOh1Fgkxfbt+CUfBqqsCQJMG9op8Os/irBPA= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= diff --git a/pkg/apiserver/rest/dualwriter.go b/pkg/apiserver/rest/dualwriter.go index 09eafb93da6..1d42944d3e9 100644 --- a/pkg/apiserver/rest/dualwriter.go +++ b/pkg/apiserver/rest/dualwriter.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" "time" "github.com/prometheus/client_golang/prometheus" @@ -306,54 +305,3 @@ func getName(o runtime.Object) string { } return accessor.GetName() } - -const dataSyncerInterval = 60 * time.Minute - -// StartPeriodicDataSyncer starts a background job that will execute the DataSyncer every 60 minutes -func StartPeriodicDataSyncer(ctx context.Context, mode DualWriterMode, legacy LegacyStorage, storage Storage, - kind string, reg prometheus.Registerer, serverLockService ServerLockService, requestInfo *request.RequestInfo) { - klog.Info("Starting periodic data syncer for mode mode: ", mode) - - // run in background - go func() { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - timeWindow := 600 // 600 seconds (10 minutes) - jitterSeconds := r.Int63n(int64(timeWindow)) - klog.Info("data syncer is going to start at: ", time.Now().Add(time.Second*time.Duration(jitterSeconds))) - time.Sleep(time.Second * time.Duration(jitterSeconds)) - - // run it immediately - syncOK, err := runDataSyncer(ctx, mode, legacy, storage, kind, reg, serverLockService, requestInfo) - klog.Info("data syncer finished, syncOK: ", syncOK, ", error: ", err) - - ticker := time.NewTicker(dataSyncerInterval) - for { - select { - case <-ticker.C: - syncOK, err = runDataSyncer(ctx, mode, legacy, storage, kind, reg, serverLockService, requestInfo) - klog.Info("data syncer finished, syncOK: ", syncOK, ", error: ", err) - case <-ctx.Done(): - return - } - } - }() -} - -// runDataSyncer will ensure that data between legacy storage and unified storage are in sync. -// The sync implementation depends on the DualWriter mode -func runDataSyncer(ctx context.Context, mode DualWriterMode, legacy LegacyStorage, storage Storage, - kind string, reg prometheus.Registerer, serverLockService ServerLockService, requestInfo *request.RequestInfo) (bool, error) { - // ensure that execution takes no longer than necessary - const timeout = dataSyncerInterval - time.Minute - ctx, cancelFn := context.WithTimeout(ctx, timeout) - defer cancelFn() - - // implementation depends on the current DualWriter mode - switch mode { - case Mode2: - return mode2DataSyncer(ctx, legacy, storage, kind, reg, serverLockService, requestInfo) - default: - klog.Info("data syncer not implemented for mode mode:", mode) - return false, nil - } -} diff --git a/pkg/apiserver/rest/dualwriter_mode2.go b/pkg/apiserver/rest/dualwriter_mode2.go index c1356c5de16..8a2a989cdb7 100644 --- a/pkg/apiserver/rest/dualwriter_mode2.go +++ b/pkg/apiserver/rest/dualwriter_mode2.go @@ -2,21 +2,16 @@ package rest import ( "context" - "fmt" "time" - "github.com/prometheus/client_golang/prometheus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/klog/v2" - "github.com/grafana/authlib/claims" - "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/utils" ) @@ -392,213 +387,3 @@ func enrichLegacyObject(originalObj, returnedObj runtime.Object) error { accessorReturned.SetUID(accessorOriginal.GetUID()) return nil } - -func getSyncRequester(orgId int64) *identity.StaticRequester { - return &identity.StaticRequester{ - Type: claims.TypeServiceAccount, // system:apiserver - UserID: 1, - OrgID: orgId, - Name: "admin", - Login: "admin", - OrgRole: identity.RoleAdmin, - IsGrafanaAdmin: true, - Permissions: map[int64]map[string][]string{ - orgId: { - "*": {"*"}, // all resources, all scopes - }, - }, - } -} - -type syncItem struct { - name string - objStorage runtime.Object - objLegacy runtime.Object -} - -func getList(ctx context.Context, obj rest.Lister, listOptions *metainternalversion.ListOptions) ([]runtime.Object, error) { - ll, err := obj.List(ctx, listOptions) - if err != nil { - return nil, err - } - - return meta.ExtractList(ll) -} - -func mode2DataSyncer(ctx context.Context, legacy LegacyStorage, storage Storage, resource string, reg prometheus.Registerer, serverLockService ServerLockService, requestInfo *request.RequestInfo) (bool, error) { - metrics := &dualWriterMetrics{} - metrics.init(reg) - - log := klog.NewKlogr().WithName("DualWriterMode2Syncer") - - everythingSynced := false - outOfSync := 0 - syncSuccess := 0 - syncErr := 0 - - maxInterval := dataSyncerInterval + 5*time.Minute - - var errSync error - const maxRecordsSync = 1000 - - // LockExecuteAndRelease ensures that just a single Grafana server acquires a lock at a time - // The parameter 'maxInterval' is a timeout safeguard, if the LastExecution in the - // database is older than maxInterval, we will assume the lock as timeouted. The 'maxInterval' parameter should be so long - // that is impossible for 2 processes to run at the same time. - err := serverLockService.LockExecuteAndRelease(ctx, "dualwriter mode 2 sync", maxInterval, func(context.Context) { - log.Info("starting dualwriter mode 2 sync") - startSync := time.Now() - - orgId := int64(1) - - ctx = klog.NewContext(ctx, log) - ctx = identity.WithRequester(ctx, getSyncRequester(orgId)) - ctx = request.WithNamespace(ctx, requestInfo.Namespace) - ctx = request.WithRequestInfo(ctx, requestInfo) - - storageList, err := getList(ctx, storage, &metainternalversion.ListOptions{ - Limit: maxRecordsSync, - }) - if err != nil { - log.Error(err, "unable to extract list from storage") - return - } - - if len(storageList) >= maxRecordsSync { - errSync = fmt.Errorf("unified storage has more than %d records. Aborting sync", maxRecordsSync) - log.Error(errSync, "Unified storage has more records to be synced than allowed") - return - } - - log.Info("got items from unified storage", "items", len(storageList)) - - legacyList, err := getList(ctx, legacy, &metainternalversion.ListOptions{}) - if err != nil { - log.Error(err, "unable to extract list from legacy storage") - return - } - log.Info("got items from legacy storage", "items", len(legacyList)) - - itemsByName := map[string]syncItem{} - for _, obj := range legacyList { - accessor, err := utils.MetaAccessor(obj) - if err != nil { - log.Error(err, "error retrieving accessor data for object from legacy storage") - continue - } - name := accessor.GetName() - - item, ok := itemsByName[name] - if !ok { - item = syncItem{} - } - item.name = name - item.objLegacy = obj - itemsByName[name] = item - } - - for _, obj := range storageList { - accessor, err := utils.MetaAccessor(obj) - if err != nil { - log.Error(err, "error retrieving accessor data for object from storage") - continue - } - name := accessor.GetName() - - item, ok := itemsByName[name] - if !ok { - item = syncItem{} - } - item.name = name - item.objStorage = obj - itemsByName[name] = item - } - log.Info("got list of items to be synced", "items", len(itemsByName)) - - for name, item := range itemsByName { - // upsert if: - // - existing in both legacy and storage, but objects are different, or - // - if it's missing from storage - if item.objLegacy != nil && - ((item.objStorage != nil && !Compare(item.objLegacy, item.objStorage)) || (item.objStorage == nil)) { - outOfSync++ - - accessor, err := utils.MetaAccessor(item.objLegacy) - if err != nil { - log.Error(err, "error retrieving accessor data for object from storage") - continue - } - - if item.objStorage != nil { - accessorStorage, err := utils.MetaAccessor(item.objStorage) - if err != nil { - log.Error(err, "error retrieving accessor data for object from storage") - continue - } - accessor.SetResourceVersion(accessorStorage.GetResourceVersion()) - accessor.SetUID(accessorStorage.GetUID()) - - log.Info("updating item on unified storage", "name", name) - } else { - accessor.SetResourceVersion("") - accessor.SetUID("") - - log.Info("inserting item on unified storage", "name", name) - } - - objInfo := rest.DefaultUpdatedObjectInfo(item.objLegacy, []rest.TransformFunc{}...) - res, _, err := storage.Update(ctx, - name, - objInfo, - func(ctx context.Context, obj runtime.Object) error { return nil }, - func(ctx context.Context, obj, old runtime.Object) error { return nil }, - true, // force creation - &metav1.UpdateOptions{}, - ) - if err != nil { - log.WithValues("object", res).Error(err, "could not update in storage") - syncErr++ - } else { - syncSuccess++ - } - } - - // delete if object does not exists on legacy but exists on storage - if item.objLegacy == nil && item.objStorage != nil { - outOfSync++ - - ctx = request.WithRequestInfo(ctx, &request.RequestInfo{ - APIGroup: requestInfo.APIGroup, - Resource: requestInfo.Resource, - Name: name, - Namespace: requestInfo.Namespace, - }) - - log.Info("deleting item from unified storage", "name", name) - - deletedS, _, err := storage.Delete(ctx, name, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{}) - if err != nil { - if !apierrors.IsNotFound(err) { - log.WithValues("objectList", deletedS).Error(err, "could not delete from storage") - } - syncErr++ - } else { - syncSuccess++ - } - } - } - - everythingSynced = outOfSync == syncSuccess - - metrics.recordDataSyncerOutcome(mode2Str, resource, everythingSynced) - metrics.recordDataSyncerDuration(err != nil, mode2Str, resource, startSync) - - log.Info("finished syncing items", "items", len(itemsByName), "updated", syncSuccess, "failed", syncErr, "outcome", everythingSynced) - }) - - if errSync != nil { - err = errSync - } - - return everythingSynced, err -} diff --git a/pkg/apiserver/rest/dualwriter_mode2_test.go b/pkg/apiserver/rest/dualwriter_mode2_test.go index 809e10fe645..fd7e168b84a 100644 --- a/pkg/apiserver/rest/dualwriter_mode2_test.go +++ b/pkg/apiserver/rest/dualwriter_mode2_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -16,7 +15,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/apis/example" - "k8s.io/apiserver/pkg/endpoints/request" ) var createFn = func(context.Context, runtime.Object) error { return nil } @@ -609,197 +607,3 @@ func TestEnrichReturnedObject(t *testing.T) { }) } } - -var legacyObj1 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo1", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} -var legacyObj2 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo2", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} -var legacyObj3 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo3", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} -var legacyObj4 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo4", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} - -var legacyObj2WithHostname = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo2", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{Hostname: "hostname"}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} - -var storageObj1 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo1", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} -var storageObj2 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo2", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} -var storageObj3 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo3", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} -var storageObj4 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo4", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} - -var legacyListWith3items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, - Items: []example.Pod{ - *legacyObj1, - *legacyObj2, - *legacyObj3, - }} - -var legacyListWith4items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, - Items: []example.Pod{ - *legacyObj1, - *legacyObj2, - *legacyObj3, - *legacyObj4, - }} - -var legacyListWith3itemsObj2IsDifferent = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, - Items: []example.Pod{ - *legacyObj1, - *legacyObj2WithHostname, - *legacyObj3, - }} - -var storageListWith3items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, - Items: []example.Pod{ - *storageObj1, - *storageObj2, - *storageObj3, - }} - -var storageListWith4items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, - Items: []example.Pod{ - *storageObj1, - *storageObj2, - *storageObj3, - *storageObj4, - }} - -var storageListWith3itemsMissingFoo2 = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, - Items: []example.Pod{ - *storageObj1, - *storageObj3, - *storageObj4, - }} - -func TestMode2_DataSyncer(t *testing.T) { - type testCase struct { - setupLegacyFn func(m *mock.Mock) - setupStorageFn func(m *mock.Mock) - name string - expectedOutcome bool - wantErr bool - } - tests := - []testCase{ - { - name: "both stores are in sync", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) - }, - expectedOutcome: true, - }, - { - name: "both stores are in sync - fail to list from legacy", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, errors.New("error")) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) - }, - expectedOutcome: false, - }, - { - name: "both stores are in sync - fail to list from storage", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, errors.New("error")) - }, - expectedOutcome: false, - }, - { - name: "storage is missing 1 entry (foo4)", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith4items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) - m.On("Update", mock.Anything, "foo4", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil) - }, - expectedOutcome: true, - }, - { - name: "storage needs to be update (foo2 is different)", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3itemsObj2IsDifferent, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) - m.On("Update", mock.Anything, "foo2", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil) - }, - expectedOutcome: true, - }, - { - name: "storage is missing 1 entry (foo4) - fail to upsert", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith4items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) - m.On("Update", mock.Anything, "foo4", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, errors.New("error")) - }, - expectedOutcome: false, - }, - { - name: "storage has an extra 1 entry (foo4)", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith4items, nil) - m.On("Delete", mock.Anything, "foo4", mock.Anything, mock.Anything).Return(exampleObj, false, nil) - }, - expectedOutcome: true, - }, - { - name: "storage has an extra 1 entry (foo4) - fail to delete", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith4items, nil) - m.On("Delete", mock.Anything, "foo4", mock.Anything, mock.Anything).Return(exampleObj, false, errors.New("error")) - }, - expectedOutcome: false, - }, - { - name: "storage is missing 1 entry (foo3) and has an extra 1 entry (foo4)", - setupLegacyFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) - }, - setupStorageFn: func(m *mock.Mock) { - m.On("List", mock.Anything, mock.Anything).Return(storageListWith3itemsMissingFoo2, nil) - m.On("Update", mock.Anything, "foo2", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil) - m.On("Delete", mock.Anything, "foo4", mock.Anything, mock.Anything).Return(exampleObj, false, nil) - }, - expectedOutcome: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - l := (LegacyStorage)(nil) - s := (Storage)(nil) - lm := &mock.Mock{} - um := &mock.Mock{} - - ls := legacyStoreMock{lm, l} - us := storageMock{um, s} - - if tt.setupLegacyFn != nil { - tt.setupLegacyFn(lm) - } - if tt.setupStorageFn != nil { - tt.setupStorageFn(um) - } - - outcome, err := mode2DataSyncer(context.Background(), ls, us, "test.kind", p, &fakeServerLock{}, &request.RequestInfo{}) - if tt.wantErr { - assert.Error(t, err) - return - } - - assert.NoError(t, err) - assert.Equal(t, tt.expectedOutcome, outcome) - }) - } -} diff --git a/pkg/apiserver/rest/dualwriter_syncer.go b/pkg/apiserver/rest/dualwriter_syncer.go new file mode 100644 index 00000000000..34bfef41a19 --- /dev/null +++ b/pkg/apiserver/rest/dualwriter_syncer.go @@ -0,0 +1,271 @@ +package rest + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/prometheus/client_golang/prometheus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/klog/v2" + + "github.com/grafana/authlib/claims" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/apimachinery/utils" +) + +type syncItem struct { + name string + objStorage runtime.Object + objLegacy runtime.Object + accessorStorage utils.GrafanaMetaAccessor + accessorLegacy utils.GrafanaMetaAccessor +} + +const dataSyncerInterval = 60 * time.Minute + +// StartPeriodicDataSyncer starts a background job that will execute the DataSyncer every 60 minutes +func StartPeriodicDataSyncer(ctx context.Context, mode DualWriterMode, legacy LegacyStorage, storage Storage, + kind string, reg prometheus.Registerer, serverLockService ServerLockService, requestInfo *request.RequestInfo) { + log := klog.NewKlogr().WithName("legacyToUnifiedStorageDataSyncer").WithValues("mode", mode, "resource", kind) + + log.Info("Starting periodic data syncer") + + // run in background + go func() { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + timeWindow := 600 // 600 seconds (10 minutes) + jitterSeconds := r.Int63n(int64(timeWindow)) + log.Info("data syncer scheduled", "starting time", time.Now().Add(time.Second*time.Duration(jitterSeconds))) + time.Sleep(time.Second * time.Duration(jitterSeconds)) + + // run it immediately + syncOK, err := runDataSyncer(ctx, mode, legacy, storage, kind, reg, serverLockService, requestInfo) + log.Info("data syncer finished", "syncOK", syncOK, "error", err) + + ticker := time.NewTicker(dataSyncerInterval) + for { + select { + case <-ticker.C: + syncOK, err = runDataSyncer(ctx, mode, legacy, storage, kind, reg, serverLockService, requestInfo) + log.Info("data syncer finished", "syncOK", syncOK, ", error", err) + case <-ctx.Done(): + return + } + } + }() +} + +// runDataSyncer will ensure that data between legacy storage and unified storage are in sync. +// The sync implementation depends on the DualWriter mode +func runDataSyncer(ctx context.Context, mode DualWriterMode, legacy LegacyStorage, storage Storage, + kind string, reg prometheus.Registerer, serverLockService ServerLockService, requestInfo *request.RequestInfo) (bool, error) { + // ensure that execution takes no longer than necessary + const timeout = dataSyncerInterval - time.Minute + ctx, cancelFn := context.WithTimeout(ctx, timeout) + defer cancelFn() + + // implementation depends on the current DualWriter mode + switch mode { + case Mode1, Mode2: + return legacyToUnifiedStorageDataSyncer(ctx, mode, legacy, storage, kind, reg, serverLockService, requestInfo) + default: + klog.Info("data syncer not implemented for mode mode:", mode) + return false, nil + } +} + +func legacyToUnifiedStorageDataSyncer(ctx context.Context, mode DualWriterMode, legacy LegacyStorage, storage Storage, resource string, reg prometheus.Registerer, serverLockService ServerLockService, requestInfo *request.RequestInfo) (bool, error) { + metrics := &dualWriterMetrics{} + metrics.init(reg) + + log := klog.NewKlogr().WithName("legacyToUnifiedStorageDataSyncer").WithValues("mode", mode, "resource", resource) + + everythingSynced := false + outOfSync := 0 + syncSuccess := 0 + syncErr := 0 + + maxInterval := dataSyncerInterval + 5*time.Minute + + var errSync error + const maxRecordsSync = 1000 + + // LockExecuteAndRelease ensures that just a single Grafana server acquires a lock at a time + // The parameter 'maxInterval' is a timeout safeguard, if the LastExecution in the + // database is older than maxInterval, we will assume the lock as timeouted. The 'maxInterval' parameter should be so long + // that is impossible for 2 processes to run at the same time. + err := serverLockService.LockExecuteAndRelease(ctx, fmt.Sprintf("legacyToUnifiedStorageDataSyncer-%d-%s", mode, resource), maxInterval, func(context.Context) { + log.Info("starting legacyToUnifiedStorageDataSyncer") + startSync := time.Now() + + orgId := int64(1) + + ctx = klog.NewContext(ctx, log) + ctx = identity.WithRequester(ctx, getSyncRequester(orgId)) + ctx = request.WithNamespace(ctx, requestInfo.Namespace) + ctx = request.WithRequestInfo(ctx, requestInfo) + + storageList, err := getList(ctx, storage, &metainternalversion.ListOptions{ + Limit: maxRecordsSync, + }) + if err != nil { + log.Error(err, "unable to extract list from storage") + return + } + + if len(storageList) >= maxRecordsSync { + errSync = fmt.Errorf("unified storage has more than %d records. Aborting sync", maxRecordsSync) + log.Error(errSync, "Unified storage has more records to be synced than allowed") + return + } + + log.Info("got items from unified storage", "items", len(storageList)) + + legacyList, err := getList(ctx, legacy, &metainternalversion.ListOptions{}) + if err != nil { + log.Error(err, "unable to extract list from legacy storage") + return + } + log.Info("got items from legacy storage", "items", len(legacyList)) + + itemsByName := map[string]syncItem{} + for _, obj := range legacyList { + accessor, err := utils.MetaAccessor(obj) + if err != nil { + log.Error(err, "error retrieving accessor data for object from legacy storage") + continue + } + name := accessor.GetName() + + item := itemsByName[name] + item.name = name + item.objLegacy = obj + item.accessorLegacy = accessor + itemsByName[name] = item + } + + for _, obj := range storageList { + accessor, err := utils.MetaAccessor(obj) + if err != nil { + log.Error(err, "error retrieving accessor data for object from storage") + continue + } + name := accessor.GetName() + + item := itemsByName[name] + item.name = name + item.objStorage = obj + item.accessorStorage = accessor + itemsByName[name] = item + } + log.Info("got list of items to be synced", "items", len(itemsByName)) + + for name, item := range itemsByName { + // upsert if: + // - existing in both legacy and storage, but objects are different, or + // - if it's missing from storage + if item.objLegacy != nil && + (item.objStorage == nil || !Compare(item.objLegacy, item.objStorage)) { + outOfSync++ + + if item.objStorage != nil { + item.accessorLegacy.SetResourceVersion(item.accessorStorage.GetResourceVersion()) + item.accessorLegacy.SetUID(item.accessorStorage.GetUID()) + + log.Info("updating item on unified storage", "name", name) + } else { + item.accessorLegacy.SetResourceVersion("") + item.accessorLegacy.SetUID("") + + log.Info("inserting item on unified storage", "name", name) + } + + objInfo := rest.DefaultUpdatedObjectInfo(item.objLegacy, []rest.TransformFunc{}...) + res, _, err := storage.Update(ctx, + name, + objInfo, + func(ctx context.Context, obj runtime.Object) error { return nil }, + func(ctx context.Context, obj, old runtime.Object) error { return nil }, + true, // force creation + &metav1.UpdateOptions{}, + ) + if err != nil { + log.WithValues("object", res).Error(err, "could not update in storage") + syncErr++ + } else { + syncSuccess++ + } + } + + // delete if object does not exists on legacy but exists on storage + if item.objLegacy == nil && item.objStorage != nil { + outOfSync++ + + ctx = request.WithRequestInfo(ctx, &request.RequestInfo{ + APIGroup: requestInfo.APIGroup, + Resource: requestInfo.Resource, + Name: name, + Namespace: requestInfo.Namespace, + }) + + log.Info("deleting item from unified storage", "name", name) + + deletedS, _, err := storage.Delete(ctx, name, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + log.WithValues("objectList", deletedS).Error(err, "could not delete from storage") + syncErr++ + continue + } + + syncSuccess++ + } + } + + everythingSynced = outOfSync == syncSuccess + + metrics.recordDataSyncerOutcome(mode, resource, everythingSynced) + metrics.recordDataSyncerDuration(err != nil, mode, resource, startSync) + + log.Info("finished syncing items", "items", len(itemsByName), "updated", syncSuccess, "failed", syncErr, "outcome", everythingSynced) + }) + + if errSync != nil { + err = errSync + } + + return everythingSynced, err +} + +func getSyncRequester(orgId int64) *identity.StaticRequester { + return &identity.StaticRequester{ + Type: claims.TypeServiceAccount, // system:apiserver + UserID: 1, + OrgID: orgId, + Name: "admin", + Login: "admin", + OrgRole: identity.RoleAdmin, + IsGrafanaAdmin: true, + Permissions: map[int64]map[string][]string{ + orgId: { + "*": {"*"}, // all resources, all scopes + }, + }, + } +} + +func getList(ctx context.Context, obj rest.Lister, listOptions *metainternalversion.ListOptions) ([]runtime.Object, error) { + ll, err := obj.List(ctx, listOptions) + if err != nil { + return nil, err + } + + return meta.ExtractList(ll) +} diff --git a/pkg/apiserver/rest/dualwriter_syncer_test.go b/pkg/apiserver/rest/dualwriter_syncer_test.go new file mode 100644 index 00000000000..ca30594c3a7 --- /dev/null +++ b/pkg/apiserver/rest/dualwriter_syncer_test.go @@ -0,0 +1,238 @@ +package rest + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/example" + "k8s.io/apiserver/pkg/endpoints/request" +) + +var legacyObj1 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo1", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} +var legacyObj2 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo2", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} +var legacyObj3 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo3", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} +var legacyObj4 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo4", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} + +var legacyObj2WithHostname = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo2", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{Hostname: "hostname"}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} + +var storageObj1 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo1", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} +var storageObj2 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo2", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} +var storageObj3 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo3", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} +var storageObj4 = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta: metav1.ObjectMeta{Name: "foo4", ResourceVersion: "1", CreationTimestamp: metav1.Time{}}, Spec: example.PodSpec{}, Status: example.PodStatus{StartTime: &metav1.Time{Time: time.Now()}}} + +var legacyListWith3items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, + Items: []example.Pod{ + *legacyObj1, + *legacyObj2, + *legacyObj3, + }} + +var legacyListWith4items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, + Items: []example.Pod{ + *legacyObj1, + *legacyObj2, + *legacyObj3, + *legacyObj4, + }} + +var legacyListWith3itemsObj2IsDifferent = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, + Items: []example.Pod{ + *legacyObj1, + *legacyObj2WithHostname, + *legacyObj3, + }} + +var storageListWith3items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, + Items: []example.Pod{ + *storageObj1, + *storageObj2, + *storageObj3, + }} + +var storageListWith4items = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, + Items: []example.Pod{ + *storageObj1, + *storageObj2, + *storageObj3, + *storageObj4, + }} + +var storageListWith3itemsMissingFoo2 = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, + Items: []example.Pod{ + *storageObj1, + *storageObj3, + *storageObj4, + }} + +func TestLegacyToUnifiedStorage_DataSyncer(t *testing.T) { + type testCase struct { + setupLegacyFn func(m *mock.Mock) + setupStorageFn func(m *mock.Mock) + name string + expectedOutcome bool + wantErr bool + } + tests := + []testCase{ + { + name: "both stores are in sync", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) + }, + expectedOutcome: true, + }, + { + name: "both stores are in sync - fail to list from legacy", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, errors.New("error")) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) + }, + expectedOutcome: false, + }, + { + name: "both stores are in sync - fail to list from storage", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, errors.New("error")) + }, + expectedOutcome: false, + }, + { + name: "storage is missing 1 entry (foo4)", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith4items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) + m.On("Update", mock.Anything, "foo4", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil) + }, + expectedOutcome: true, + }, + { + name: "storage needs to be update (foo2 is different)", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3itemsObj2IsDifferent, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) + m.On("Update", mock.Anything, "foo2", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil) + }, + expectedOutcome: true, + }, + { + name: "storage is missing 1 entry (foo4) - fail to upsert", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith4items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3items, nil) + m.On("Update", mock.Anything, "foo4", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, errors.New("error")) + }, + expectedOutcome: false, + }, + { + name: "storage has an extra 1 entry (foo4)", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith4items, nil) + m.On("Delete", mock.Anything, "foo4", mock.Anything, mock.Anything).Return(exampleObj, false, nil) + }, + expectedOutcome: true, + }, + { + name: "storage has an extra 1 entry (foo4) - fail to delete", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith4items, nil) + m.On("Delete", mock.Anything, "foo4", mock.Anything, mock.Anything).Return(exampleObj, false, errors.New("error")) + }, + expectedOutcome: false, + }, + { + name: "storage is missing 1 entry (foo3) and has an extra 1 entry (foo4)", + setupLegacyFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(legacyListWith3items, nil) + }, + setupStorageFn: func(m *mock.Mock) { + m.On("List", mock.Anything, mock.Anything).Return(storageListWith3itemsMissingFoo2, nil) + m.On("Update", mock.Anything, "foo2", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil) + m.On("Delete", mock.Anything, "foo4", mock.Anything, mock.Anything).Return(exampleObj, false, nil) + }, + expectedOutcome: true, + }, + } + + // mode 1 + for _, tt := range tests { + t.Run("Mode-1-"+tt.name, func(t *testing.T) { + l := (LegacyStorage)(nil) + s := (Storage)(nil) + lm := &mock.Mock{} + um := &mock.Mock{} + + ls := legacyStoreMock{lm, l} + us := storageMock{um, s} + + if tt.setupLegacyFn != nil { + tt.setupLegacyFn(lm) + } + if tt.setupStorageFn != nil { + tt.setupStorageFn(um) + } + + outcome, err := legacyToUnifiedStorageDataSyncer(context.Background(), Mode1, ls, us, "test.kind", p, &fakeServerLock{}, &request.RequestInfo{}) + if tt.wantErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutcome, outcome) + }) + } + + // mode 2 + for _, tt := range tests { + t.Run("Mode-2-"+tt.name, func(t *testing.T) { + l := (LegacyStorage)(nil) + s := (Storage)(nil) + lm := &mock.Mock{} + um := &mock.Mock{} + + ls := legacyStoreMock{lm, l} + us := storageMock{um, s} + + if tt.setupLegacyFn != nil { + tt.setupLegacyFn(lm) + } + if tt.setupStorageFn != nil { + tt.setupStorageFn(um) + } + + outcome, err := legacyToUnifiedStorageDataSyncer(context.Background(), Mode2, ls, us, "test.kind", p, &fakeServerLock{}, &request.RequestInfo{}) + if tt.wantErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutcome, outcome) + }) + } +} diff --git a/pkg/apiserver/rest/metrics.go b/pkg/apiserver/rest/metrics.go index 780b9b8a4ea..436f6a7dabe 100644 --- a/pkg/apiserver/rest/metrics.go +++ b/pkg/apiserver/rest/metrics.go @@ -1,6 +1,7 @@ package rest import ( + "fmt" "strconv" "time" @@ -97,15 +98,15 @@ func (m *dualWriterMetrics) recordOutcome(mode string, name string, areEqual boo m.outcome.WithLabelValues(mode, name, method).Observe(observeValue) } -func (m *dualWriterMetrics) recordDataSyncerDuration(isError bool, mode string, resource string, startFrom time.Time) { +func (m *dualWriterMetrics) recordDataSyncerDuration(isError bool, mode DualWriterMode, resource string, startFrom time.Time) { duration := time.Since(startFrom).Seconds() - m.syncer.WithLabelValues(strconv.FormatBool(isError), mode, resource).Observe(duration) + m.syncer.WithLabelValues(strconv.FormatBool(isError), fmt.Sprintf("%d", mode), resource).Observe(duration) } -func (m *dualWriterMetrics) recordDataSyncerOutcome(mode string, resource string, synced bool) { +func (m *dualWriterMetrics) recordDataSyncerOutcome(mode DualWriterMode, resource string, synced bool) { var observeValue float64 if !synced { observeValue = 1 } - m.syncerOutcome.WithLabelValues(mode, resource).Observe(observeValue) + m.syncerOutcome.WithLabelValues(fmt.Sprintf("%d", mode), resource).Observe(observeValue) } diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go index fa98316e9c2..7b775ad0f44 100644 --- a/pkg/build/cmd.go +++ b/pkg/build/cmd.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" "time" + + "github.com/urfave/cli/v2" ) const ( @@ -30,6 +32,12 @@ func logError(message string, err error) int { return 1 } +func RunCmdCLI(c *cli.Context) error { + os.Exit(RunCmd()) + + return nil +} + // RunCmd runs the build command and returns the exit code func RunCmd() int { opts := BuildOptsFromFlags() diff --git a/pkg/build/cmd/argcount_wrapper.go b/pkg/build/cmd/argcount_wrapper.go index 690695cd350..394283e163d 100644 --- a/pkg/build/cmd/argcount_wrapper.go +++ b/pkg/build/cmd/argcount_wrapper.go @@ -2,20 +2,6 @@ package main import "github.com/urfave/cli/v2" -// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided. -func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc { - return func(ctx *cli.Context) error { - if ctx.NArg() != num { - if err := cli.ShowSubcommandHelp(ctx); err != nil { - return cli.Exit(err.Error(), 1) - } - return cli.Exit("", 1) - } - - return action(ctx) - } -} - // ArgCountWrapper will cause the action to fail if there were more than `num` args provided. func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { return func(ctx *cli.Context) error { diff --git a/pkg/build/cmd/buildbackend.go b/pkg/build/cmd/buildbackend.go deleted file mode 100644 index 40c4f90388a..00000000000 --- a/pkg/build/cmd/buildbackend.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "fmt" - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/compilers" - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/grafana" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildBackend(ctx *cli.Context) error { - metadata, err := config.GenerateMetadata(ctx) - if err != nil { - return err - } - version := metadata.GrafanaVersion - - var ( - edition = config.Edition(ctx.String("edition")) - cfg = config.Config{ - NumWorkers: ctx.Int("jobs"), - } - ) - - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return fmt.Errorf("could not get version / package info for mode '%s': %w", metadata.ReleaseMode.Mode, err) - } - - const grafanaDir = "." - - log.Printf("Building Grafana back-end, version %q, %s edition, variants [%v]", - version, edition, buildConfig.Variants) - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - if err := compilers.Install(); err != nil { - return cli.Exit(err.Error(), 1) - } - - g, _ := errutil.GroupWithContext(ctx.Context) - for _, variant := range buildConfig.Variants { - variant := variant - - opts := grafana.BuildVariantOpts{ - Variant: variant, - Edition: edition, - Version: version, - GrafanaDir: grafanaDir, - } - - p.Schedule(g.Wrap(func() error { - return grafana.BuildVariant(ctx.Context, opts) - })) - } - if err := g.Wait(); err != nil { - return cli.Exit(err.Error(), 1) - } - - log.Println("Successfully built back-end binaries!") - return nil -} diff --git a/pkg/build/cmd/builddocker.go b/pkg/build/cmd/builddocker.go deleted file mode 100644 index f623057ee60..00000000000 --- a/pkg/build/cmd/builddocker.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/docker" - "github.com/grafana/grafana/pkg/build/gcloud" -) - -func BuildDocker(c *cli.Context) error { - if err := docker.Init(); err != nil { - return err - } - - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - useUbuntu := c.Bool("ubuntu") - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return err - } - - shouldSave := buildConfig.Docker.ShouldSave - if shouldSave { - if err := gcloud.ActivateServiceAccount(); err != nil { - return err - } - } - - edition := config.Edition(c.String("edition")) - - version := metadata.GrafanaVersion - - log.Printf("Building Docker images, version %s, %s edition, Ubuntu based: %v...", version, edition, - useUbuntu) - - for _, arch := range buildConfig.Docker.Architectures { - if _, err := docker.BuildImage(version, arch, ".", useUbuntu, shouldSave, edition, metadata.ReleaseMode.Mode); err != nil { - return cli.Exit(err.Error(), 1) - } - } - - log.Println("Successfully built Docker images!") - return nil -} diff --git a/pkg/build/cmd/buildfrontend.go b/pkg/build/cmd/buildfrontend.go deleted file mode 100644 index ef0fbc9a5fd..00000000000 --- a/pkg/build/cmd/buildfrontend.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/frontend" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildFrontend(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - cfg, mode, err := frontend.GetConfig(c, metadata) - if err != nil { - return err - } - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - g, _ := errutil.GroupWithContext(c.Context) - if err := frontend.Build(mode, frontend.GrafanaDir, p, g); err != nil { - return err - } - if err := g.Wait(); err != nil { - return err - } - - log.Println("Successfully built Grafana front-end!") - - return nil -} diff --git a/pkg/build/cmd/buildfrontendpackages.go b/pkg/build/cmd/buildfrontendpackages.go deleted file mode 100644 index b1fef3dfcee..00000000000 --- a/pkg/build/cmd/buildfrontendpackages.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "log" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/frontend" - "github.com/grafana/grafana/pkg/build/syncutil" - "github.com/urfave/cli/v2" -) - -func BuildFrontendPackages(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - cfg, mode, err := frontend.GetConfig(c, metadata) - if err != nil { - return err - } - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - g, _ := errutil.GroupWithContext(c.Context) - if err := frontend.BuildFrontendPackages(cfg.PackageVersion, mode, frontend.GrafanaDir, p, g); err != nil { - return cli.Exit(err.Error(), 1) - } - if err := g.Wait(); err != nil { - return cli.Exit(err.Error(), 1) - } - - log.Println("Successfully built Grafana front-end packages!") - - return nil -} diff --git a/pkg/build/cmd/buildinternalplugins.go b/pkg/build/cmd/buildinternalplugins.go deleted file mode 100644 index 15a1397158d..00000000000 --- a/pkg/build/cmd/buildinternalplugins.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "context" - "log" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/plugins" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildInternalPlugins(c *cli.Context) error { - cfg := config.Config{ - NumWorkers: c.Int("jobs"), - } - - const grafanaDir = "." - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return err - } - - log.Println("Building internal Grafana plug-ins...") - - ctx := context.Background() - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - var g *errutil.Group - g, ctx = errutil.GroupWithContext(ctx) - if err := plugins.Build(ctx, grafanaDir, p, g, buildConfig); err != nil { - return cli.Exit(err.Error(), 1) - } - if err := g.Wait(); err != nil { - return cli.Exit(err.Error(), 1) - } - - if err := plugins.Download(ctx, grafanaDir, p); err != nil { - return cli.Exit(err.Error(), 1) - } - - log.Println("Successfully built Grafana plug-ins!") - - return nil -} diff --git a/pkg/build/cmd/enterprisecheck.go b/pkg/build/cmd/enterprisecheck.go deleted file mode 100644 index 3fb13f6c893..00000000000 --- a/pkg/build/cmd/enterprisecheck.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "strconv" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/env" - "github.com/grafana/grafana/pkg/build/git" -) - -// checkOpts are options used to create a new GitHub check for the enterprise downstream test. -type checkOpts struct { - SHA string - URL string - Branch string - PR int -} - -func getCheckOpts(args []string) (*checkOpts, error) { - branch, ok := env.Lookup("DRONE_SOURCE_BRANCH", args) - if !ok { - return nil, cli.Exit("Unable to retrieve build source branch", 1) - } - - var ( - rgx = git.PRCheckRegexp() - matches = rgx.FindStringSubmatch(branch) - ) - - sha, ok := env.Lookup("SOURCE_COMMIT", args) - if !ok { - if matches == nil || len(matches) <= 1 { - return nil, cli.Exit("Unable to retrieve source commit", 1) - } - sha = matches[2] - } - - url, ok := env.Lookup("DRONE_BUILD_LINK", args) - if !ok { - return nil, cli.Exit(`missing environment variable "DRONE_BUILD_LINK"`, 1) - } - - prStr, ok := env.Lookup("OSS_PULL_REQUEST", args) - if !ok { - if matches == nil || len(matches) <= 1 { - return nil, cli.Exit("Unable to retrieve PR number", 1) - } - - prStr = matches[1] - } - - pr, err := strconv.Atoi(prStr) - if err != nil { - return nil, err - } - - return &checkOpts{ - Branch: branch, - PR: pr, - SHA: sha, - URL: url, - }, nil -} - -// EnterpriseCheckBegin creates the GitHub check and signals the beginning of the downstream build / test process -func EnterpriseCheckBegin(c *cli.Context) error { - var ( - ctx = c.Context - client = git.NewGitHubClient(ctx, c.String("github-token")) - ) - - opts, err := getCheckOpts(os.Environ()) - if err != nil { - return err - } - - if _, err = git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, "pending"); err != nil { - return err - } - - return nil -} - -func EnterpriseCheckSuccess(c *cli.Context) error { - return completeEnterpriseCheck(c, true) -} - -func EnterpriseCheckFail(c *cli.Context) error { - return completeEnterpriseCheck(c, false) -} - -func completeEnterpriseCheck(c *cli.Context, success bool) error { - var ( - ctx = c.Context - client = git.NewGitHubClient(ctx, c.String("github-token")) - ) - - // Update the pull request labels - opts, err := getCheckOpts(os.Environ()) - if err != nil { - return err - } - - status := "failure" - if success { - status = "success" - } - - // Update the GitHub check... - if _, err := git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, status); err != nil { - return err - } - - // Delete branch if needed - log.Printf("Checking branch '%s' against '%s'", git.PRCheckRegexp().String(), opts.Branch) - if git.PRCheckRegexp().MatchString(opts.Branch) { - log.Println("Deleting branch", opts.Branch) - if err := git.DeleteEnterpriseBranch(ctx, client.Git, opts.Branch); err != nil { - return fmt.Errorf("error deleting enterprise branch: %w", err) - } - } - - label := "enterprise-failed" - if success { - label = "enterprise-ok" - } - - return git.AddLabelToPR(ctx, client.Issues, opts.PR, label) -} diff --git a/pkg/build/cmd/enterprisecheck_test.go b/pkg/build/cmd/enterprisecheck_test.go deleted file mode 100644 index 0eeb5bd5741..00000000000 --- a/pkg/build/cmd/enterprisecheck_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGetCheckOpts(t *testing.T) { - t.Run("it should return the checkOpts if the correct environment variables are set", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - "OSS_PULL_REQUEST=1", - } - - opts, err := getCheckOpts(args) - require.NoError(t, err) - require.Equal(t, opts.SHA, "1234") - require.Equal(t, opts.URL, "http://example.com") - }) - t.Run("it should return an error if SOURCE_COMMIT is not set", func(t *testing.T) { - args := []string{ - "DRONE_BUILD_LINK=http://example.com", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - "OSS_PULL_REQUEST=1", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) - t.Run("it should return an error if DRONE_BUILD_LINK is not set", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "OSS_PULL_REQUEST=1", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) - t.Run("it should return an error if OSS_PULL_REQUEST is not set", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) - t.Run("it should return an error if OSS_PULL_REQUEST is not an integer", func(t *testing.T) { - args := []string{ - "SOURCE_COMMIT=1234", - "DRONE_SOURCE_BRANCH=test", - "DRONE_BUILD_LINK=http://example.com", - "OSS_PULL_REQUEST=http://example.com", - } - - opts, err := getCheckOpts(args) - require.Nil(t, opts) - require.Error(t, err) - }) -} diff --git a/pkg/build/cmd/exportversion.go b/pkg/build/cmd/exportversion.go deleted file mode 100644 index c4c4744ebef..00000000000 --- a/pkg/build/cmd/exportversion.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" -) - -func ExportVersion(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - const distDir = "dist" - if err := os.RemoveAll(distDir); err != nil { - return err - } - if err := os.Mkdir(distDir, 0750); err != nil { - return err - } - - // nolint:gosec - if err := os.WriteFile(filepath.Join(distDir, "grafana.version"), []byte(metadata.GrafanaVersion), 0664); err != nil { - return err - } - - return nil -} diff --git a/pkg/build/cmd/fetchimages.go b/pkg/build/cmd/fetchimages.go index b131cc9a73d..26260aedb19 100644 --- a/pkg/build/cmd/fetchimages.go +++ b/pkg/build/cmd/fetchimages.go @@ -4,7 +4,7 @@ import ( "fmt" "log" "os/exec" - "strings" + "path/filepath" "github.com/urfave/cli/v2" @@ -18,6 +18,25 @@ const ( ubuntu = "ubuntu" ) +// GetImageFiles returns the list of image (.img, but should be .tar because they are tar archives) files that are +// created in the 'tag' process and stored in the prerelease bucket, waiting to be released. +func GetImageFiles(grafana string, version string, architectures []config.Architecture) []string { + bases := []string{alpine, ubuntu} + images := []string{} + for _, base := range bases { + for _, arch := range architectures { + image := fmt.Sprintf("%s-%s-%s.img", grafana, version, arch) + if base == "ubuntu" { + image = fmt.Sprintf("%s-%s-ubuntu-%s.img", grafana, version, arch) + } + + images = append(images, image) + } + } + + return images +} + func FetchImages(c *cli.Context) error { if c.NArg() > 0 { if err := cli.ShowSubcommandHelp(c); err != nil { @@ -44,74 +63,65 @@ func FetchImages(c *cli.Context) error { Tag: metadata.GrafanaVersion, } - edition := fmt.Sprintf("-%s", cfg.Edition) + grafana := "grafana" + if cfg.Edition == "enterprise" { + grafana = "grafana-enterprise" + } + if cfg.Edition == "enterprise2" { + grafana = "grafana-enterprise2" + } + if cfg.Edition == "grafana" || cfg.Edition == "oss" { + grafana = "grafana-oss" + } - err = gcloud.ActivateServiceAccount() - if err != nil { + baseURL := fmt.Sprintf("gs://%s/%s/", cfg.Bucket, cfg.Tag) + images := GetImageFiles(grafana, cfg.Tag, cfg.Archs) + + log.Printf("Fetching images [%v]", images) + + if err := gcloud.ActivateServiceAccount(); err != nil { return err } - - var basesStr []string - for _, base := range cfg.Distribution { - switch base { - case alpine: - basesStr = append(basesStr, "") - case ubuntu: - basesStr = append(basesStr, "-ubuntu") - default: - return fmt.Errorf("unrecognized base %q", base) - } - } - - err = downloadFromGCS(cfg, basesStr, edition) - if err != nil { + if err := DownloadImages(baseURL, images, "."); err != nil { return err } - - err = loadImages(cfg, basesStr, edition) - if err != nil { + if err := LoadImages(images, "."); err != nil { return err } return nil } -func loadImages(cfg docker.Config, basesStr []string, edition string) error { - log.Println("Loading fetched image files to local docker registry...") - log.Printf("Number of images to be loaded: %d\n", len(basesStr)*len(cfg.Archs)) - for _, base := range basesStr { - for _, arch := range cfg.Archs { - imageFilename := fmt.Sprintf("grafana%s-%s%s-%s.img", edition, cfg.Tag, base, arch) - log.Printf("image file name: %s\n", imageFilename) - //nolint:gosec - cmd := exec.Command("docker", "load", "-i", imageFilename) - cmd.Dir = "." - out, err := cmd.CombinedOutput() - if err != nil { - log.Printf("out: %s\n", out) - return fmt.Errorf("error loading image: %q", err) - } - log.Printf("Successfully loaded %s!\n %s\n", fmt.Sprintf("grafana%s-%s%s-%s", edition, cfg.Tag, base, arch), out) +// LoadImages uses the `docker load -i` command to load the image tar file into the docker daemon so that it can be +// tagged and pushed. +func LoadImages(images []string, source string) error { + p := filepath.Clean(source) + for _, image := range images { + image := filepath.Join(p, image) + log.Println("Loading image", image) + //nolint:gosec + cmd := exec.Command("docker", "load", "-i", image) + cmd.Dir = "." + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("out: %s\n", out) + return fmt.Errorf("error loading image: %q", err) } + log.Println("Loaded image", image) } log.Println("Images successfully loaded!") return nil } -func downloadFromGCS(cfg docker.Config, basesStr []string, edition string) error { - log.Printf("Downloading Docker images from GCS bucket: %s\n", cfg.Bucket) - - for _, base := range basesStr { - for _, arch := range cfg.Archs { - src := fmt.Sprintf("gs://%s/%s/grafana%s-%s%s-%s.img", cfg.Bucket, cfg.Tag, edition, cfg.Tag, base, arch) - args := strings.Split(fmt.Sprintf("-m cp -r %s .", src), " ") - //nolint:gosec - cmd := exec.Command("gsutil", args...) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to download: %w\n%s", err, out) - } +func DownloadImages(baseURL string, images []string, destination string) error { + for _, image := range images { + p := baseURL + image + log.Println("Downloading image", p) + //nolint:gosec + cmd := exec.Command("gsutil", "-m", "cp", "-r", p, destination) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to download: %w\n%s", err, out) } } - log.Printf("Successfully fetched image files from %s bucket!\n", cfg.Bucket) return nil } diff --git a/pkg/build/cmd/fetchimages_test.go b/pkg/build/cmd/fetchimages_test.go new file mode 100644 index 00000000000..9452b011a66 --- /dev/null +++ b/pkg/build/cmd/fetchimages_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/config" + "github.com/stretchr/testify/require" +) + +func TestGetImageFiles(t *testing.T) { + var ( + architectures = []config.Architecture{ + config.ArchAMD64, + config.ArchARM64, + config.ArchARMv7, + } + ) + + t.Run("1.2.3", func(t *testing.T) { + expect := []string{ + "grafana-oss-1.2.3-amd64.img", + "grafana-oss-1.2.3-arm64.img", + "grafana-oss-1.2.3-armv7.img", + "grafana-oss-1.2.3-ubuntu-amd64.img", + "grafana-oss-1.2.3-ubuntu-arm64.img", + "grafana-oss-1.2.3-ubuntu-armv7.img", + } + + res := GetImageFiles("grafana-oss", "1.2.3", architectures) + + require.Equal(t, expect, res) + }) + + t.Run("1.2.3+example-01", func(t *testing.T) { + expect := []string{ + "grafana-oss-1.2.3+example-01-amd64.img", + "grafana-oss-1.2.3+example-01-arm64.img", + "grafana-oss-1.2.3+example-01-armv7.img", + "grafana-oss-1.2.3+example-01-ubuntu-amd64.img", + "grafana-oss-1.2.3+example-01-ubuntu-arm64.img", + "grafana-oss-1.2.3+example-01-ubuntu-armv7.img", + } + + res := GetImageFiles("grafana-oss", "1.2.3+example-01", architectures) + + require.Equal(t, expect, res) + }) +} diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go index c9020fe2403..7ab2eb6f993 100644 --- a/pkg/build/cmd/flags.go +++ b/pkg/build/cmd/flags.go @@ -16,37 +16,15 @@ var ( Usage: "The edition of Grafana to build (oss or enterprise)", Value: "oss", } - variantsFlag = cli.StringFlag{ - Name: "variants", - Usage: "Comma-separated list of variants to build", - } triesFlag = cli.IntFlag{ Name: "tries", Usage: "Specify number of tries before failing", Value: 1, } - noInstallDepsFlag = cli.BoolFlag{ - Name: "no-install-deps", - Usage: "Don't install dependencies", - } - signingAdminFlag = cli.BoolFlag{ - Name: "signing-admin", - Usage: "Use manifest signing admin API endpoint?", - } - signFlag = cli.BoolFlag{ - Name: "sign", - Usage: "Enable plug-in signing (you must set GRAFANA_API_KEY)", - } dryRunFlag = cli.BoolFlag{ Name: "dry-run", Usage: "Only simulate actions", } - gitHubTokenFlag = cli.StringFlag{ - Name: "github-token", - Value: "", - EnvVars: []string{"GITHUB_TOKEN"}, - Usage: "GitHub token", - } tagFlag = cli.StringFlag{ Name: "tag", Usage: "Grafana version tag", diff --git a/pkg/build/cmd/grafanacom.go b/pkg/build/cmd/grafanacom.go index 0627effcefa..305aa1195e8 100644 --- a/pkg/build/cmd/grafanacom.go +++ b/pkg/build/cmd/grafanacom.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/gcloud" "github.com/grafana/grafana/pkg/build/gcloud/storage" + "github.com/grafana/grafana/pkg/build/gcom" "github.com/grafana/grafana/pkg/build/packaging" ) @@ -125,6 +126,48 @@ func getReleaseURLs() (string, string, error) { return pconf.Grafana.WhatsNewURL, pconf.Grafana.ReleaseNotesURL, nil } +func Builds(baseURL *url.URL, grafana, version string, packages []packaging.BuildArtifact) ([]GCOMPackage, error) { + builds := make([]GCOMPackage, len(packages)) + for i, v := range packages { + var ( + os = v.Distro + arch = v.Arch + ) + + if v.Distro == "windows" { + os = "win" + if v.Ext == "msi" { + os = "win-installer" + } + } + + if v.Distro == "rhel" { + if arch == "aarch64" { + arch = "arm64" + } + } + + if v.Distro == "deb" { + if arch == "armhf" { + arch = "armv7" + if v.RaspberryPi { + log.Println(v.Distro, arch, "raspberrypi == true") + arch = "armv6" + } + } + } + + u := gcom.GetURL(baseURL, version, grafana, v.Distro, v.Arch, v.Ext, v.Musl, v.RaspberryPi) + builds[i] = GCOMPackage{ + OS: os, + URL: u.String(), + Arch: arch, + } + } + + return builds, nil +} + // publishPackages publishes packages to grafana.com. func publishPackages(cfg packaging.PublishConfig) error { log.Printf("Publishing Grafana packages, version %s, %s edition, %s mode, dryRun: %v, simulating: %v...\n", @@ -133,14 +176,17 @@ func publishPackages(cfg packaging.PublishConfig) error { versionStr := fmt.Sprintf("v%s", cfg.Version) log.Printf("Creating release %s at grafana.com...\n", versionStr) - var sfx string - var pth string + var ( + pth string + grafana = "grafana" + ) + switch cfg.Edition { case config.EditionOSS: pth = "oss" case config.EditionEnterprise: + grafana = "grafana-enterprise" pth = "enterprise" - sfx = packaging.EnterpriseSfx default: return fmt.Errorf("unrecognized edition %q", cfg.Edition) } @@ -152,28 +198,19 @@ func publishPackages(cfg packaging.PublishConfig) error { pth = path.Join(pth, packaging.ReleaseFolder) } - product := fmt.Sprintf("grafana%s", sfx) - pth = path.Join(pth, product) - baseArchiveURL := fmt.Sprintf("https://dl.grafana.com/%s", pth) - - builds := make([]buildRepr, len(packaging.ArtifactConfigs)) - for i, ba := range packaging.ArtifactConfigs { - u := ba.GetURL(baseArchiveURL, cfg) - - sha256, err := getSHA256(u) - if err != nil { - return err - } - - builds[i] = buildRepr{ - OS: ba.Os, - URL: u, - SHA256: string(sha256), - Arch: ba.Arch, - } + pth = path.Join(pth) + baseArchiveURL := &url.URL{ + Scheme: "https", + Host: "dl.grafana.com", + Path: pth, } - r := releaseRepr{ + builds, err := Builds(baseArchiveURL, grafana, cfg.Version, packaging.ArtifactConfigs) + if err != nil { + return err + } + + r := Release{ Version: cfg.Version, ReleaseDate: time.Now().UTC(), Builds: builds, @@ -195,6 +232,15 @@ func publishPackages(cfg packaging.PublishConfig) error { return err } + for i, v := range r.Builds { + sha, err := getSHA256(v.URL) + if err != nil { + return err + } + + r.Builds[i].SHA256 = string(sha) + } + for _, b := range r.Builds { if err := postRequest(cfg, fmt.Sprintf("versions/%s/packages", cfg.Version), b, fmt.Sprintf("create build %s %s", b.OS, b.Arch)); err != nil { @@ -211,6 +257,7 @@ func publishPackages(cfg packaging.PublishConfig) error { func getSHA256(u string) ([]byte, error) { shaURL := fmt.Sprintf("%s.sha256", u) + // nolint:gosec resp, err := http.Get(shaURL) if err != nil { @@ -232,7 +279,7 @@ func getSHA256(u string) ([]byte, error) { return sha256, nil } -func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) error { +func postRequest(cfg packaging.PublishConfig, pth string, body any, descr string) error { var sfx string switch cfg.Edition { case config.EditionOSS: @@ -243,7 +290,7 @@ func postRequest(cfg packaging.PublishConfig, pth string, obj any, descr string) } product := fmt.Sprintf("grafana%s", sfx) - jsonB, err := json.Marshal(obj) + jsonB, err := json.Marshal(body) if err != nil { return fmt.Errorf("failed to JSON encode release: %w", err) } @@ -303,20 +350,20 @@ func constructURL(product string, pth string) (string, error) { return u.String(), err } -type buildRepr struct { +type GCOMPackage struct { OS string `json:"os"` URL string `json:"url"` SHA256 string `json:"sha256"` Arch string `json:"arch"` } -type releaseRepr struct { - Version string `json:"version"` - ReleaseDate time.Time `json:"releaseDate"` - Stable bool `json:"stable"` - Beta bool `json:"beta"` - Nightly bool `json:"nightly"` - WhatsNewURL string `json:"whatsNewUrl"` - ReleaseNotesURL string `json:"releaseNotesUrl"` - Builds []buildRepr `json:"-"` +type Release struct { + Version string `json:"version"` + ReleaseDate time.Time `json:"releaseDate"` + Stable bool `json:"stable"` + Beta bool `json:"beta"` + Nightly bool `json:"nightly"` + WhatsNewURL string `json:"whatsNewUrl"` + ReleaseNotesURL string `json:"releaseNotesUrl"` + Builds []GCOMPackage `json:"-"` } diff --git a/pkg/build/cmd/grafanacom_test.go b/pkg/build/cmd/grafanacom_test.go index bf90874b1c9..9f4fbd7c981 100644 --- a/pkg/build/cmd/grafanacom_test.go +++ b/pkg/build/cmd/grafanacom_test.go @@ -1,7 +1,14 @@ package main import ( + "fmt" + "net/url" + "path" "testing" + + "github.com/grafana/grafana/pkg/build/packaging" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_constructURL(t *testing.T) { @@ -33,3 +40,221 @@ func Test_constructURL(t *testing.T) { }) } } + +func TestBuilds(t *testing.T) { + baseURL := &url.URL{ + Scheme: "https", + Host: "dl.example.com", + Path: path.Join("oss", "release"), + } + + version := "1.2.3" + grafana := "grafana" + packages := []packaging.BuildArtifact{ + { + Distro: "deb", + Arch: "arm64", + Ext: "deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm", + }, + { + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + RaspberryPi: true, + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + }, + { + Distro: "linux", + Arch: "armv7", + Ext: "tar.gz", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "zip", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "msi", + }, + } + + expect := []GCOMPackage{ + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3_arm64.deb", + OS: "deb", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3-1.aarch64.rpm", + OS: "rhel", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.linux-arm64.tar.gz", + OS: "linux", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-rpi_1.2.3_armhf.deb", + OS: "deb", + Arch: "armv6", + }, + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3_armhf.deb", + OS: "deb", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.linux-armv7.tar.gz", + OS: "linux", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.windows-amd64.zip", + OS: "win", + Arch: "amd64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3.windows-amd64.msi", + OS: "win-installer", + Arch: "amd64", + }, + } + + builds, err := Builds(baseURL, grafana, version, packages) + require.NoError(t, err) + require.Equal(t, len(expect), len(builds)) + + for i := range builds { + t.Run(fmt.Sprintf("[%d/%d] %s", i+1, len(builds), expect[i].URL), func(t *testing.T) { + assert.Equal(t, expect[i].URL, builds[i].URL) + assert.Equal(t, expect[i].OS, builds[i].OS) + assert.Equal(t, expect[i].Arch, builds[i].Arch) + }) + } +} + +func TestBuildsWithPlus(t *testing.T) { + baseURL := &url.URL{ + Scheme: "https", + Host: "dl.example.com", + Path: path.Join("oss", "release"), + } + + version := "1.2.3+example-01" + grafana := "grafana" + packages := []packaging.BuildArtifact{ + { + Distro: "deb", + Arch: "arm64", + Ext: "deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm", + }, + { + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + RaspberryPi: true, + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + }, + { + Distro: "linux", + Arch: "armv7", + Ext: "tar.gz", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "zip", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "msi", + }, + } + + expect := []GCOMPackage{ + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3+example~01_arm64.deb", + OS: "deb", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example~01-1.aarch64.rpm", + OS: "rhel", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.linux-arm64.tar.gz", + OS: "linux", + Arch: "arm64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-rpi_1.2.3+example~01_armhf.deb", + OS: "deb", + Arch: "armv6", + }, + { + URL: "https://dl.example.com/oss/release/grafana_1.2.3+example~01_armhf.deb", + OS: "deb", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.linux-armv7.tar.gz", + OS: "linux", + Arch: "armv7", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.windows-amd64.zip", + OS: "win", + Arch: "amd64", + }, + { + URL: "https://dl.example.com/oss/release/grafana-1.2.3+example-01.windows-amd64.msi", + OS: "win-installer", + Arch: "amd64", + }, + } + + builds, err := Builds(baseURL, grafana, version, packages) + require.NoError(t, err) + require.Equal(t, len(expect), len(builds)) + + for i := range builds { + t.Run(fmt.Sprintf("[%d/%d] %s", i+1, len(builds), expect[i].URL), func(t *testing.T) { + assert.Equal(t, expect[i].URL, builds[i].URL) + assert.Equal(t, expect[i].OS, builds[i].OS) + assert.Equal(t, expect[i].Arch, builds[i].Arch) + }) + } +} diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index dff9db8f8b0..d207abe020b 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -3,11 +3,9 @@ package main import ( "log" "os" - "strings" + "github.com/grafana/grafana/pkg/build" "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/docker" ) var additionalCommands []*cli.Command = make([]*cli.Command, 0, 5) @@ -21,28 +19,8 @@ func main() { app := cli.NewApp() app.Commands = cli.Commands{ { - Name: "build-backend", - Usage: "Build one or more variants of back-end binaries", - ArgsUsage: "[version]", - Action: MaxArgCountWrapper(1, BuildBackend), - Flags: []cli.Flag{ - &jobsFlag, - &variantsFlag, - &editionFlag, - &buildIDFlag, - }, - }, - { - Name: "build-frontend-packages", - Usage: "Build front-end packages", - ArgsUsage: "[version]", - Action: BuildFrontendPackages, - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &buildIDFlag, - &noInstallDepsFlag, - }, + Name: "build", + Action: build.RunCmdCLI, }, { Name: "e2e-tests", @@ -71,44 +49,11 @@ func main() { }, }, }, - { - Name: "build-frontend", - Usage: "Build front-end artifacts", - ArgsUsage: "[version]", - Action: MaxArgCountWrapper(1, BuildFrontend), - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &buildIDFlag, - }, - }, { Name: "whatsnew-checker", Usage: "Checks whatsNewUrl in package.json for differences between the tag and the docs version", Action: WhatsNewChecker, }, - { - Name: "build-docker", - Usage: "Build Grafana Docker images", - Action: MaxArgCountWrapper(1, BuildDocker), - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &cli.BoolFlag{ - Name: "ubuntu", - Usage: "Use Ubuntu base image", - }, - &cli.BoolFlag{ - Name: "shouldSave", - Usage: "Should save docker image to tarball", - }, - &cli.StringFlag{ - Name: "archs", - Value: strings.Join(docker.AllArchs, ","), - Usage: "Comma separated architectures to build", - }, - }, - }, { Name: "upload-cdn", Usage: "Upload public/* to a cdn bucket", @@ -117,23 +62,6 @@ func main() { &editionFlag, }, }, - { - Name: "shellcheck", - Usage: "Run shellcheck on shell scripts", - Action: Shellcheck, - }, - { - Name: "build-plugins", - Usage: "Build internal plug-ins", - Action: MaxArgCountWrapper(1, BuildInternalPlugins), - Flags: []cli.Flag{ - &jobsFlag, - &editionFlag, - &signingAdminFlag, - &signFlag, - &noInstallDepsFlag, - }, - }, { Name: "publish-metrics", Usage: "Publish a set of metrics from stdin", @@ -145,30 +73,6 @@ func main() { Usage: "Verify Drone configuration", Action: VerifyDrone, }, - { - Name: "verify-starlark", - Usage: "Verify Starlark configuration", - ArgsUsage: "", - Action: VerifyStarlark, - }, - { - Name: "export-version", - Usage: "Exports version in dist/grafana.version", - Action: ExportVersion, - }, - { - Name: "package", - Usage: "Package one or more Grafana variants", - ArgsUsage: "[version]", - Action: MaxArgCountWrapper(1, Package), - Flags: []cli.Flag{ - &jobsFlag, - &variantsFlag, - &editionFlag, - &buildIDFlag, - &signFlag, - }, - }, { Name: "store-storybook", Usage: "Stores storybook to GCS buckets", @@ -279,18 +183,6 @@ func main() { &editionFlag, }, }, - { - Name: "publish-enterprise2", - Usage: "Handle Grafana Enterprise2 Docker images", - ArgsUsage: "[version]", - Action: Enterprise2, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "dockerhub-repo", - Usage: "DockerHub repo to push images", - }, - }, - }, }, }, { @@ -399,36 +291,6 @@ func main() { }, }, }, - { - Name: "enterprise-check", - Usage: "Commands for testing against Grafana Enterprise", - Subcommands: cli.Commands{ - { - Name: "begin", - Usage: "Creates the GitHub check in a pull request and begins the tests", - Action: EnterpriseCheckBegin, - Flags: []cli.Flag{ - &gitHubTokenFlag, - }, - }, - { - Name: "success", - Usage: "Updates the GitHub check in a pull request to show a successful build and updates the pull request labels", - Action: EnterpriseCheckSuccess, - Flags: []cli.Flag{ - &gitHubTokenFlag, - }, - }, - { - Name: "fail", - Usage: "Updates the GitHub check in a pull request to show a failed build and updates the pull request labels", - Action: EnterpriseCheckFail, - Flags: []cli.Flag{ - &gitHubTokenFlag, - }, - }, - }, - }, } app.Commands = append(app.Commands, additionalCommands...) diff --git a/pkg/build/cmd/package.go b/pkg/build/cmd/package.go deleted file mode 100644 index 6b961512ec1..00000000000 --- a/pkg/build/cmd/package.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "context" - "log" - "strings" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/gpg" - "github.com/grafana/grafana/pkg/build/packaging" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func Package(c *cli.Context) error { - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - edition := config.Edition(c.String("edition")) - - releaseMode, err := metadata.GetReleaseMode() - if err != nil { - return cli.Exit(err.Error(), 1) - } - - releaseModeConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return cli.Exit(err.Error(), 1) - } - - cfg := config.Config{ - NumWorkers: c.Int("jobs"), - SignPackages: c.Bool("sign"), - } - - ctx := context.Background() - - variants := []config.Variant{} - variantStrs := strings.Split(c.String("variants"), ",") - if c.String("variants") != "" { - for _, varStr := range variantStrs { - if varStr == "" { - continue - } - variants = append(variants, config.Variant(varStr)) - } - } else { - variants = releaseModeConfig.Variants - } - - if len(variants) == 0 { - variants = config.AllVariants - } - - log.Printf("Packaging Grafana version %q, version mode %s, %s edition, variants %s", metadata.GrafanaVersion, releaseMode.Mode, - edition, strings.Join(variantStrs, ",")) - - if cfg.SignPackages { - if err := gpg.LoadGPGKeys(&cfg); err != nil { - return cli.Exit(err, 1) - } - defer gpg.RemoveGPGFiles(cfg) - if err := gpg.Import(cfg); err != nil { - return cli.Exit(err, 1) - } - } - - p := syncutil.NewWorkerPool(cfg.NumWorkers) - defer p.Close() - - if err := packaging.PackageGrafana(ctx, metadata.GrafanaVersion, ".", cfg, edition, variants, releaseModeConfig.PluginSignature.Sign, p); err != nil { - return cli.Exit(err, 1) - } - - log.Println("Successfully packaged Grafana!") - return nil -} diff --git a/pkg/build/cmd/publishimages_enterprise2.go b/pkg/build/cmd/publishimages_enterprise2.go deleted file mode 100644 index 94cd4b57c7d..00000000000 --- a/pkg/build/cmd/publishimages_enterprise2.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/docker" - "github.com/grafana/grafana/pkg/build/gcloud" -) - -func Enterprise2(c *cli.Context) error { - if c.NArg() > 0 { - if err := cli.ShowSubcommandHelp(c); err != nil { - return cli.Exit(err.Error(), 1) - } - return cli.Exit("", 1) - } - - if err := gcloud.ActivateServiceAccount(); err != nil { - return fmt.Errorf("couldn't activate service account, err: %w", err) - } - - metadata, err := config.GenerateMetadata(c) - if err != nil { - return err - } - - buildConfig, err := config.GetBuildConfig(metadata.ReleaseMode.Mode) - if err != nil { - return err - } - - cfg := docker.Config{ - Archs: buildConfig.Docker.Architectures, - Distribution: buildConfig.Docker.Distribution, - DockerHubRepo: c.String("dockerhub-repo"), - Tag: metadata.GrafanaVersion, - } - - err = dockerLoginEnterprise2() - if err != nil { - return err - } - - var distributionStr []string - for _, distribution := range cfg.Distribution { - switch distribution { - case alpine: - distributionStr = append(distributionStr, "") - case ubuntu: - distributionStr = append(distributionStr, "-ubuntu") - default: - return fmt.Errorf("unrecognized distribution %q", distribution) - } - } - - for _, distribution := range distributionStr { - var imageFileNames []string - for _, arch := range cfg.Archs { - imageFilename := fmt.Sprintf("%s:%s%s-%s", cfg.DockerHubRepo, cfg.Tag, distribution, arch) - err := docker.PushImage(imageFilename) - if err != nil { - return err - } - imageFileNames = append(imageFileNames, imageFilename) - } - manifest := fmt.Sprintf("%s:%s%s", cfg.DockerHubRepo, cfg.Tag, distribution) - args := []string{"manifest", "create", manifest} - args = append(args, imageFileNames...) - - //nolint:gosec - cmd := exec.Command("docker", args...) - cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to create Docker manifest: %w\n%s", err, output) - } - - err = docker.PushManifest(manifest) - if err != nil { - return err - } - } - - return nil -} - -func dockerLoginEnterprise2() error { - log.Println("Docker login...") - cmd := exec.Command("gcloud", "auth", "configure-docker") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("error logging in to DockerHub: %s %q", out, err) - } - - log.Println("Successful login!") - return nil -} diff --git a/pkg/build/cmd/shellcheck.go b/pkg/build/cmd/shellcheck.go deleted file mode 100644 index e18b1d85258..00000000000 --- a/pkg/build/cmd/shellcheck.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/urfave/cli/v2" -) - -func Shellcheck(c *cli.Context) error { - log.Println("Running shellcheck...") - - fpaths := []string{} - if err := filepath.Walk("scripts", func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if strings.HasSuffix(path, ".sh") { - fpaths = append(fpaths, path) - } - - return nil - }); err != nil { - return fmt.Errorf("couldn't traverse scripts/: %w", err) - } - - log.Printf("Running shellcheck on %s", strings.Join(fpaths, ",")) - args := append([]string{"-e", "SC1071", "-e", "SC2162"}, fpaths...) - //nolint:gosec - cmd := exec.Command("shellcheck", args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("shellcheck failed: %s", output) - } - - log.Println("Successfully ran shellcheck!") - return nil -} diff --git a/pkg/build/cmd/uploadpackages.go b/pkg/build/cmd/uploadpackages.go index da0dd6a607a..5801192c62c 100644 --- a/pkg/build/cmd/uploadpackages.go +++ b/pkg/build/cmd/uploadpackages.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/urfave/cli/v2" @@ -14,9 +15,28 @@ import ( "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/droneutil" "github.com/grafana/grafana/pkg/build/gcloud" - "github.com/grafana/grafana/pkg/build/packaging" ) +// PackageRegexp returns a regexp for matching packages corresponding to a certain Grafana edition. +func PackageRegexp(edition config.Edition) *regexp.Regexp { + var sfx string + switch edition { + case config.EditionOSS: + case config.EditionEnterprise: + sfx = "-enterprise" + case config.EditionEnterprise2: + sfx = "-enterprise2" + default: + panic(fmt.Sprintf("unrecognized edition %q", edition)) + } + rePkg, err := regexp.Compile(fmt.Sprintf(`^grafana%s(?:-rpi)?[-_][^-_]+.*$`, sfx)) + if err != nil { + panic(fmt.Sprintf("Failed to compile regexp: %s", err)) + } + + return rePkg +} + const releaseFolder = "release" const mainFolder = "main" const releaseBranchFolder = "prerelease" @@ -181,7 +201,7 @@ func uploadPackages(cfg uploadConfig) error { return fmt.Errorf("failed to list packages: %w", err) } fpaths := []string{} - rePkg := packaging.PackageRegexp(cfg.edition) + rePkg := PackageRegexp(cfg.edition) for _, fpath := range matches { fname := filepath.Base(fpath) if strings.Contains(fname, "latest") || !rePkg.MatchString(fname) { diff --git a/pkg/build/cmd/verifystarlark.go b/pkg/build/cmd/verifystarlark.go deleted file mode 100644 index ff33a77a3af..00000000000 --- a/pkg/build/cmd/verifystarlark.go +++ /dev/null @@ -1,142 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os/exec" - "path/filepath" - "strings" - - "github.com/urfave/cli/v2" -) - -func mapSlice[I any, O any](a []I, f func(I) O) []O { - o := make([]O, len(a)) - for i, e := range a { - o[i] = f(e) - } - return o -} - -// VerifyStarlark is the CLI Action for verifying Starlark files in a workspace. -// It expects a single context argument which is the path to the workspace. -// The actual verification procedure can return multiple errors which are -// joined together to be one holistic error for the action. -func VerifyStarlark(c *cli.Context) error { - if c.NArg() != 1 { - var message string - if c.NArg() == 0 { - message = "ERROR: missing required argument " - } - if c.NArg() > 1 { - message = "ERROR: too many arguments" - } - - if err := cli.ShowSubcommandHelp(c); err != nil { - return err - } - - return cli.Exit(message, 1) - } - - workspace := c.Args().Get(0) - verificationErrs, executionErr := verifyStarlark(c.Context, workspace, buildifierLintCommand) - if executionErr != nil { - return executionErr - } - - if len(verificationErrs) == 0 { - return nil - } - - noun := "file" - if len(verificationErrs) > 1 { - noun += "s" - } - - return fmt.Errorf("verification failed for %d %s:\n%s", - len(verificationErrs), - noun, - strings.Join( - mapSlice(verificationErrs, func(e error) string { return e.Error() }), - "\n", - )) -} - -type commandFunc = func(path string) (command string, args []string) - -func buildifierLintCommand(path string) (string, []string) { - return "buildifier", []string{"-lint", "warn", "-mode", "check", path} -} - -// verifyStarlark walks all directories starting at provided workspace path and -// verifies any Starlark files it finds. -// Starlark files are assumed to end with the .star extension. -// The verification relies on linting frovided by the 'buildifier' binary which -// must be in the PATH. -// A slice of verification errors are returned, one for each file that failed verification. -// If any execution of the `buildifier` command fails, this is returned separately. -// commandFn is executed on every Starlark file to determine the command and arguments to be executed. -// The caller is trusted and it is the callers responsibility to ensure that the resulting command is safe to execute. -func verifyStarlark(ctx context.Context, workspace string, commandFn commandFunc) ([]error, error) { - var verificationErrs []error - - // All errors from filepath.WalkDir are filtered by the fs.WalkDirFunc. - // Lstat or ReadDir errors are reported as verificationErrors. - // If any execution of the `buildifier` command fails or if the context is cancelled, - // it is reported as an error and any verification of subsequent files is skipped. - err := filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { - // Skip verification of the file or files within the directory if there is an error - // returned by Lstat or ReadDir. - if err != nil { - verificationErrs = append(verificationErrs, err) - return nil - } - - if d.IsDir() { - return nil - } - - if filepath.Ext(path) == ".star" { - command, args := commandFn(path) - // The caller is trusted. - //nolint:gosec - cmd := exec.CommandContext(ctx, command, args...) - cmd.Dir = workspace - - _, err = cmd.Output() - if err == nil { // No error, early return. - return nil - } - - // The error returned from cmd.Output() is never wrapped. - //nolint:errorlint - if err, ok := err.(*exec.ExitError); ok { - switch err.ExitCode() { - // Case comments are informed by the output of `buildifier --help` - case 1: // syntax errors in input - verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) - return nil - case 2: // usage errors: invoked incorrectly - return fmt.Errorf("command %q: %s", cmd, err.Stderr) - case 3: // unexpected runtime errors: file I/O problems or internal bugs - return fmt.Errorf("command %q: %s", cmd, err.Stderr) - case 4: // check mode failed (reformat is needed) - verificationErrs = append(verificationErrs, errors.New(string(err.Stderr))) - return nil - default: - return fmt.Errorf("command %q: %s", cmd, err.Stderr) - } - } - - // Error was not an exit error from the command. - return fmt.Errorf("command %q: %v", cmd, err) - } - - return nil - }) - - return verificationErrs, err -} diff --git a/pkg/build/cmd/verifystarlark_test.go b/pkg/build/cmd/verifystarlark_test.go deleted file mode 100644 index 1121ca5bd64..00000000000 --- a/pkg/build/cmd/verifystarlark_test.go +++ /dev/null @@ -1,137 +0,0 @@ -//go:build requires_buildifier - -package main - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestVerifyStarlark(t *testing.T) { - t.Run("execution errors", func(t *testing.T) { - t.Run("invalid usage", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - - _, executionErr := verifyStarlark(ctx, workspace, func(string) (string, []string) { return "buildifier", []string{"--invalid"} }) - if executionErr == nil { - t.Fatalf("Expected execution error but got none") - } - }) - - t.Run("context cancellation", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - workspace := t.TempDir() - err := os.WriteFile(filepath.Join(workspace, "ignored.star"), []byte{}, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - err = os.WriteFile(filepath.Join(workspace, "other-ignored.star"), []byte{}, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - cancel() - - _, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr == nil { - t.Fatalf("Expected execution error but got none") - } - }) - }) - - t.Run("verification errors", func(t *testing.T) { - t.Run("a single file with lint", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - - invalidContent := []byte(`load("scripts/drone/other.star", "function") - -function()`) - err := os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm) - if err != nil { - t.Fatalf(err.Error()) - } - - verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr != nil { - t.Fatalf("Unexpected execution error: %v", executionErr) - } - if len(verificationErrs) == 0 { - t.Fatalf(`"has-lint.star" requires linting but the verifyStarlark function provided no linting error`) - } - if len(verificationErrs) > 1 { - t.Fatalf(`verifyStarlark returned multiple errors for the "has-lint.star" file but only one was expected: %v`, verificationErrs) - } - if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { - t.Fatalf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) - } - }) - - t.Run("no files with lint", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - - content := []byte(`""" -This module does nothing. -""" - -load("scripts/drone/other.star", "function") - -function() -`) - require.NoError(t, os.WriteFile(filepath.Join(workspace, "no-lint.star"), content, os.ModePerm)) - - verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr != nil { - t.Fatalf("Unexpected execution error: %v", executionErr) - } - if len(verificationErrs) != 0 { - t.Log(`"no-lint.star" has no lint but the verifyStarlark function provided at least one error`) - for _, err := range verificationErrs { - t.Log(err) - } - t.FailNow() - } - }) - - t.Run("multiple files with lint", func(t *testing.T) { - ctx := context.Background() - workspace := t.TempDir() - - invalidContent := []byte(`load("scripts/drone/other.star", "function") - -function()`) - require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint.star"), invalidContent, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(workspace, "has-lint2.star"), invalidContent, os.ModePerm)) - - verificationErrs, executionErr := verifyStarlark(ctx, workspace, buildifierLintCommand) - if executionErr != nil { - t.Fatalf("Unexpected execution error: %v", executionErr) - } - if len(verificationErrs) == 0 { - t.Fatalf(`Two files require linting but the verifyStarlark function provided no linting error`) - } - if len(verificationErrs) == 1 { - t.Fatalf(`Two files require linting but the verifyStarlark function provided only one linting error: %v`, verificationErrs[0]) - } - if len(verificationErrs) > 2 { - t.Fatalf(`verifyStarlark returned more errors than expected: %v`, verificationErrs) - } - if !strings.Contains(verificationErrs[0].Error(), "has-lint.star:1: module-docstring: The file has no module docstring.") { - t.Errorf(`"has-lint.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) - } - if !strings.Contains(verificationErrs[1].Error(), "has-lint2.star:1: module-docstring: The file has no module docstring.") { - t.Fatalf(`"has-lint2.star" is missing a module docstring but the verifyStarlark function linting error did not mention this, instead we got: %v`, verificationErrs[0]) - } - }) - }) -} diff --git a/pkg/build/compilers/install.go b/pkg/build/compilers/install.go deleted file mode 100644 index 9a45ba48c0b..00000000000 --- a/pkg/build/compilers/install.go +++ /dev/null @@ -1,50 +0,0 @@ -package compilers - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" -) - -const ( - ArmV6 = "/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc" - Armv7 = "arm-linux-gnueabihf-gcc" - Armv7Musl = "/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc" - Arm64 = "aarch64-linux-gnu-gcc" - Arm64Musl = "/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc" - Osx64 = "/tmp/osxcross/target/bin/o64-clang" - Win64 = "x86_64-w64-mingw32-gcc" - LinuxX64 = "/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc" - LinuxX64Musl = "/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc" -) - -func Install() error { - // From the os.TempDir documentation: - // On Unix systems, it returns $TMPDIR if non-empty, - // else /tmp. On Windows, it uses GetTempPath, - // returning the first non-empty value from %TMP%, %TEMP%, %USERPROFILE%, - // or the Windows directory. On Plan 9, it returns /tmp. - tmp := os.TempDir() - - var ( - centosArchive = "x86_64-centos6-linux-gnu.tar.xz" - osxArchive = "osxcross.tar.xz" - ) - - for _, fname := range []string{centosArchive, osxArchive} { - path := filepath.Join(tmp, fname) - if _, err := os.Stat(path); err != nil { - return fmt.Errorf("stat error: %w", err) - } - // Ignore gosec G204 as this function is only used in the build process. - //nolint:gosec - cmd := exec.Command("tar", "xfJ", fname) - cmd.Dir = tmp - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to unpack %q: %q, %w", fname, output, err) - } - } - - return nil -} diff --git a/pkg/build/config/revision.go b/pkg/build/config/revision.go deleted file mode 100644 index 70be21cdf18..00000000000 --- a/pkg/build/config/revision.go +++ /dev/null @@ -1,69 +0,0 @@ -package config - -import ( - "context" - "fmt" - "log" - "strconv" - "time" - - "github.com/grafana/grafana/pkg/build/executil" -) - -type Revision struct { - Timestamp int64 - SHA256 string - EnterpriseCommit string - Branch string -} - -func GrafanaTimestamp(ctx context.Context, dir string) (int64, error) { - out, err := executil.OutputAt(ctx, dir, "git", "show", "-s", "--format=%ct") - if err != nil { - return time.Now().Unix(), nil - } - - stamp, err := strconv.ParseInt(out, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse output from git show: %q", out) - } - - return stamp, nil -} - -// GrafanaRevision uses git commands to get information about the checked out Grafana code located at 'grafanaDir'. -// This could maybe be a more generic "Describe" function in the "git" package. -func GrafanaRevision(ctx context.Context, grafanaDir string) (Revision, error) { - stamp, err := GrafanaTimestamp(ctx, grafanaDir) - if err != nil { - return Revision{}, err - } - - sha, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--short", "HEAD") - if err != nil { - return Revision{}, err - } - - enterpriseCommit, err := executil.OutputAt(ctx, grafanaDir, "git", "-C", "../grafana-enterprise", "rev-parse", "--short", "HEAD") - if err != nil { - enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "..", "rev-parse", "--short", "HEAD") - if err != nil { - enterpriseCommit, err = executil.OutputAt(ctx, grafanaDir, "git", "-C", "/tmp/grafana-enterprise", "rev-parse", "--short", "HEAD") - if err != nil { - log.Println("Could not get enterprise commit. Error:", err) - } - } - } - - branch, err := executil.OutputAt(ctx, grafanaDir, "git", "rev-parse", "--abbrev-ref", "HEAD") - if err != nil { - return Revision{}, err - } - - return Revision{ - SHA256: sha, - EnterpriseCommit: enterpriseCommit, - Branch: branch, - Timestamp: stamp, - }, nil -} diff --git a/pkg/build/cryptoutil/md5.go b/pkg/build/cryptoutil/md5.go deleted file mode 100644 index fa4b4fcc5ec..00000000000 --- a/pkg/build/cryptoutil/md5.go +++ /dev/null @@ -1,35 +0,0 @@ -package cryptoutil - -import ( - "crypto/md5" - "fmt" - "io" - "log" - "os" -) - -func MD5File(fpath string) error { - // Ignore gosec G304 as this function is only used in the build process. - //nolint:gosec - fd, err := os.Open(fpath) - if err != nil { - return err - } - defer func() { - if err := fd.Close(); err != nil { - log.Printf("error closing file at '%s': %s", fpath, err.Error()) - } - }() - - h := md5.New() // nolint:gosec - if _, err = io.Copy(h, fd); err != nil { - return err - } - - // nolint:gosec - if err := os.WriteFile(fpath+".md5", []byte(fmt.Sprintf("%x\n", h.Sum(nil))), 0664); err != nil { - return err - } - - return nil -} diff --git a/pkg/build/docker/build.go b/pkg/build/docker/build.go deleted file mode 100644 index 8731ab7a247..00000000000 --- a/pkg/build/docker/build.go +++ /dev/null @@ -1,182 +0,0 @@ -package docker - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/grafana/grafana/pkg/build/config" -) - -// verifyArchive verifies the integrity of an archive file. -func verifyArchive(archive string) error { - log.Printf("Verifying checksum of %q", archive) - - //nolint:gosec - shaB, err := os.ReadFile(archive + ".sha256") - if err != nil { - return err - } - - exp := strings.TrimSpace(string(shaB)) - - //nolint:gosec - f, err := os.Open(archive) - if err != nil { - return err - } - - defer func() { - if err := f.Close(); err != nil { - log.Println("error closing file:", err) - } - }() - - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return err - } - - chksum := hex.EncodeToString(h.Sum(nil)) - if chksum != exp { - return fmt.Errorf("archive checksum is different than expected: %q", archive) - } - - log.Printf("Archive %q has expected checksum: %s", archive, exp) - - return nil -} - -// BuildImage builds a Docker image. -// The image tag is returned. -func BuildImage(version string, arch config.Architecture, grafanaDir string, useUbuntu, shouldSave bool, edition config.Edition, mode config.VersionMode) ([]string, error) { - var baseArch string - - switch arch { - case "amd64": - case "armv7": - baseArch = "arm32v7/" - case "arm64": - baseArch = "arm64v8/" - default: - return []string{}, fmt.Errorf("unrecognized architecture %q", arch) - } - - libc := "-musl" - baseImage := fmt.Sprintf("%salpine:3.18.5", baseArch) - tagSuffix := "" - if useUbuntu { - libc = "" - baseImage = fmt.Sprintf("%subuntu:22.04", baseArch) - tagSuffix = "-ubuntu" - } - - var editionStr string - var dockerRepo string - var additionalDockerRepo string - var tags []string - var imageFileBase string - var dockerEnterprise2Repo string - if repo, ok := os.LookupEnv("DOCKER_ENTERPRISE2_REPO"); ok { - dockerEnterprise2Repo = repo - } - - switch edition { - case config.EditionOSS: - dockerRepo = "grafana/grafana-image-tags" - additionalDockerRepo = "grafana/grafana-oss-image-tags" - imageFileBase = "grafana-oss" - case config.EditionEnterprise: - dockerRepo = "grafana/grafana-enterprise-image-tags" - imageFileBase = "grafana-enterprise" - editionStr = "-enterprise" - case config.EditionEnterprise2: - dockerRepo = dockerEnterprise2Repo - imageFileBase = "grafana-enterprise2" - editionStr = "-enterprise2" - default: - return []string{}, fmt.Errorf("unrecognized edition %s", edition) - } - - buildDir := filepath.Join(grafanaDir, "packaging/docker") - // For example: grafana-8.5.0-52819pre.linux-amd64-musl.tar.gz - archive := fmt.Sprintf("grafana%s-%s.linux-%s%s.tar.gz", editionStr, version, arch, libc) - if err := verifyArchive(filepath.Join(buildDir, archive)); err != nil { - return []string{}, err - } - - tag := fmt.Sprintf("%s:%s%s-%s", dockerRepo, version, tagSuffix, arch) - tags = append(tags, tag) - - args := []string{ - "build", - "-q", - "--build-arg", fmt.Sprintf("BASE_IMAGE=%s", baseImage), - "--build-arg", fmt.Sprintf("GRAFANA_TGZ=%s", archive), - "--build-arg", "GO_SRC=tgz-builder", - "--build-arg", "JS_SRC=tgz-builder", - "--build-arg", "RUN_SH=./run.sh", - "--tag", tag, - "--no-cache", - "--file", "../../Dockerfile", - ".", - "--label", fmt.Sprintf("mode=%s", string(mode)), - } - - //nolint:gosec - cmd := exec.Command("docker", args...) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled", "DOCKER_BUILDKIT=1") - log.Printf("Running Docker: DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=1 %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("building Docker image failed: %w", err) - } - if shouldSave { - imageFile := fmt.Sprintf("%s-%s%s-%s.img", imageFileBase, version, tagSuffix, arch) - //nolint:gosec - cmd = exec.Command("docker", "save", tag, "-o", imageFile) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("Running Docker: %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("saving Docker image failed: %w", err) - } - gcsURL := fmt.Sprintf("gs://grafana-prerelease/artifacts/docker/%s/%s", version, imageFile) - //nolint:gosec - cmd = exec.Command("gsutil", "-q", "cp", imageFile, gcsURL) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("Running gsutil: %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("storing Docker image failed: %w", err) - } - log.Printf("Docker image %s stored to grafana-prerelease GCS bucket", imageFile) - } - if additionalDockerRepo != "" { - additionalTag := fmt.Sprintf("%s:%s%s-%s", additionalDockerRepo, version, tagSuffix, arch) - - //nolint:gosec - cmd = exec.Command("docker", "tag", tag, additionalTag) - cmd.Dir = buildDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - log.Printf("Running Docker: %s", cmd) - if err := cmd.Run(); err != nil { - return []string{}, fmt.Errorf("tagging Docker image failed: %w", err) - } - tags = append(tags, additionalTag) - } - - return tags, nil -} diff --git a/pkg/build/docker/init.go b/pkg/build/docker/init.go deleted file mode 100644 index 94fe236aa52..00000000000 --- a/pkg/build/docker/init.go +++ /dev/null @@ -1,34 +0,0 @@ -package docker - -import ( - "fmt" - "log" - "os" - "os/exec" -) - -// AllArchs is a list of all supported Docker image architectures. -var AllArchs = []string{"amd64", "arm64"} - -// emulatorImage is the docker image used as the cross-platform emulator -var emulatorImage = "tonistiigi/binfmt:qemu-v7.0.0" - -// Init initializes the OS for Docker image building. -func Init() error { - // Necessary for cross-platform builds - if err := os.Setenv("DOCKER_BUILDKIT", "1"); err != nil { - log.Println("error setting DOCKER_BUILDKIT environment variable:", err) - } - - // Enable execution of Docker images for other architectures - //nolint:gosec - cmd := exec.Command("docker", "run", "--privileged", "--rm", - emulatorImage, "--install", "all") - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to enable execution of cross-platform Docker images: %w\n%s", err, output) - } - log.Println("emulators have been installed successfully!") - - return nil -} diff --git a/pkg/build/docker/push.go b/pkg/build/docker/push.go deleted file mode 100644 index da87ef2cb16..00000000000 --- a/pkg/build/docker/push.go +++ /dev/null @@ -1,62 +0,0 @@ -package docker - -import ( - "fmt" - "log" - "os" - "os/exec" - "time" -) - -const ( - tries = 3 - sleepTime = 30 -) - -func PushImage(newImage string) error { - var err error - for i := 0; i < tries; i++ { - log.Printf("push attempt #%d...", i+1) - var out []byte - cmd := exec.Command("docker", "push", newImage) - cmd.Dir = "." - out, err = cmd.CombinedOutput() - if err != nil { - log.Printf("output: %s", out) - log.Printf("sleep for %d, before retrying...", sleepTime) - time.Sleep(sleepTime * time.Second) - } else { - log.Printf("Successfully pushed %s!", newImage) - break - } - } - if err != nil { - return fmt.Errorf("error pushing images to DockerHub: %q", err) - } - return nil -} - -func PushManifest(manifest string) error { - log.Printf("Pushing Docker manifest %s...", manifest) - - var err error - for i := 0; i < tries; i++ { - log.Printf("push attempt #%d...", i+1) - var out []byte - cmd := exec.Command("docker", "manifest", "push", manifest) - cmd.Env = append(os.Environ(), "DOCKER_CLI_EXPERIMENTAL=enabled") - out, err = cmd.CombinedOutput() - if err != nil { - log.Printf("output: %s", out) - log.Printf("sleep for %d, before retrying...", sleepTime) - time.Sleep(sleepTime * time.Second) - } else { - log.Printf("Successful manifest push! %s", string(out)) - break - } - } - if err != nil { - return fmt.Errorf("failed to push manifest, err: %w", err) - } - return nil -} diff --git a/pkg/build/env/lookup.go b/pkg/build/env/lookup.go deleted file mode 100644 index 993b7259e14..00000000000 --- a/pkg/build/env/lookup.go +++ /dev/null @@ -1,18 +0,0 @@ -package env - -import ( - "strings" -) - -// Lookup is the equivalent of os.LookupEnv, only you are able to provide the list of environment variables. -// To use this as os.LookupEnv would be used, simply call -// `env.Lookup("ENVIRONMENT_VARIABLE", os.Environ())` -func Lookup(name string, vars []string) (string, bool) { - for _, v := range vars { - if strings.HasPrefix(v, name) { - return strings.TrimPrefix(v, name+"="), true - } - } - - return "", false -} diff --git a/pkg/build/env/lookup_test.go b/pkg/build/env/lookup_test.go deleted file mode 100644 index ccd44b11e8c..00000000000 --- a/pkg/build/env/lookup_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package env_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/build/env" -) - -func TestLookup(t *testing.T) { - values := []string{"ENV_1=a", "ENV_2=b", "ENV_3=c", "ENV_4_TEST="} - - { - v, ok := env.Lookup("ENV_1", values) - require.Equal(t, v, "a") - require.True(t, ok) - } - - { - v, ok := env.Lookup("ENV_2", values) - require.Equal(t, v, "b") - require.True(t, ok) - } - - { - v, ok := env.Lookup("ENV_3", values) - require.Equal(t, v, "c") - require.True(t, ok) - } - - { - v, ok := env.Lookup("ENV_4_TEST", values) - require.Equal(t, v, "") - require.True(t, ok) - } - - { - v, ok := env.Lookup("NOT_THERE", values) - require.Equal(t, v, "") - require.False(t, ok) - } -} diff --git a/pkg/build/errutil/group.go b/pkg/build/errutil/group.go deleted file mode 100644 index 0844616e579..00000000000 --- a/pkg/build/errutil/group.go +++ /dev/null @@ -1,61 +0,0 @@ -package errutil - -import ( - "context" - "log" - "sync" -) - -type Group struct { - cancel func() - wg sync.WaitGroup - errOnce sync.Once - err error -} - -func GroupWithContext(ctx context.Context) (*Group, context.Context) { - ctx, cancel := context.WithCancel(ctx) - return &Group{cancel: cancel}, ctx -} - -// Wait waits for any wrapped goroutines to finish and returns any error having occurred in one of them. -func (g *Group) Wait() error { - log.Println("Waiting on Group") - g.wg.Wait() - if g.cancel != nil { - log.Println("Group canceling its context after waiting") - g.cancel() - } - return g.err -} - -// Cancel cancels the associated context. -func (g *Group) Cancel() { - log.Println("Group's Cancel method being called") - g.cancel() -} - -// Wrap wraps a function to be executed in a goroutine. -func (g *Group) Wrap(f func() error) func() { - g.wg.Add(1) - return func() { - defer g.wg.Done() - - if err := f(); err != nil { - g.errOnce.Do(func() { - log.Printf("An error occurred in Group: %s", err) - g.err = err - if g.cancel != nil { - log.Println("Group canceling its context due to error") - g.cancel() - } - }) - } - } -} - -// Go wraps the provided function and executes it in a goroutine. -func (g *Group) Go(f func() error) { - wrapped := g.Wrap(f) - go wrapped() -} diff --git a/pkg/build/executil/exec.go b/pkg/build/executil/exec.go deleted file mode 100644 index e46f568a997..00000000000 --- a/pkg/build/executil/exec.go +++ /dev/null @@ -1,46 +0,0 @@ -package executil - -import ( - "context" - "fmt" - "os/exec" - "strings" -) - -func RunAt(ctx context.Context, dir, cmd string, args ...string) error { - // Ignore gosec G204 as this function is only used in the build process. - //nolint:gosec - c := exec.CommandContext(ctx, cmd, args...) - c.Dir = dir - - b, err := c.CombinedOutput() - - if err != nil { - return fmt.Errorf("%w. '%s %v': %s", err, cmd, args, string(b)) - } - - return nil -} - -func Run(ctx context.Context, cmd string, args ...string) error { - return RunAt(ctx, ".", cmd, args...) -} - -func OutputAt(ctx context.Context, dir, cmd string, args ...string) (string, error) { - // Ignore gosec G204 as this function is only used in the build process. - //nolint:gosec - c := exec.CommandContext(ctx, cmd, args...) - c.Dir = dir - - b, err := c.CombinedOutput() - - if err != nil { - return "", err - } - - return strings.TrimSpace(string(b)), nil -} - -func Output(ctx context.Context, cmd string, args ...string) (string, error) { - return OutputAt(ctx, ".", cmd, args...) -} diff --git a/pkg/build/frontend/build.go b/pkg/build/frontend/build.go deleted file mode 100644 index c4259c25e57..00000000000 --- a/pkg/build/frontend/build.go +++ /dev/null @@ -1,56 +0,0 @@ -package frontend - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/lerna" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -func BuildFrontendPackages(version string, edition config.Edition, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group) error { - p.Schedule(g.Wrap(func() error { - if err := lerna.BuildFrontendPackages(version, edition, grafanaDir); err != nil { - return fmt.Errorf("failed to build %s frontend packages: %v", edition, err) - } - - log.Printf("Finished building %s frontend packages", string(edition)) - return nil - })) - - return nil -} - -// Build builds the Grafana front-end -func Build(edition config.Edition, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group) error { - log.Printf("Building %s frontend in %q", edition, grafanaDir) - grafanaDir, err := filepath.Abs(grafanaDir) - if err != nil { - return err - } - - for _, dpath := range []string{"tmp", "public_gen", "public/build"} { - dpath = filepath.Join(grafanaDir, dpath) - if err := os.RemoveAll(dpath); err != nil { - return fmt.Errorf("failed to remove %q: %w", dpath, err) - } - } - - p.Schedule(g.Wrap(func() error { - cmd := exec.Command("yarn", "run", "build") - cmd.Dir = grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to build %s frontend with webpack: %s", edition, output) - } - - log.Printf("Finished building %s frontend", edition) - return nil - })) - - return nil -} diff --git a/pkg/build/frontend/config.go b/pkg/build/frontend/config.go deleted file mode 100644 index 318168de1d4..00000000000 --- a/pkg/build/frontend/config.go +++ /dev/null @@ -1,42 +0,0 @@ -package frontend - -import ( - "fmt" - - "github.com/blang/semver/v4" - "github.com/grafana/grafana/pkg/build/config" - "github.com/urfave/cli/v2" -) - -const GrafanaDir = "." - -func GetConfig(c *cli.Context, metadata config.Metadata) (config.Config, config.Edition, error) { - cfg := config.Config{ - NumWorkers: c.Int("jobs"), - GitHubToken: c.String("github-token"), - } - - mode := config.Edition(c.String("edition")) - - if metadata.ReleaseMode.Mode == config.TagMode && !metadata.ReleaseMode.IsTest { - packageJSONVersion, err := config.GetPackageJSONVersion(GrafanaDir) - if err != nil { - return config.Config{}, "", err - } - semverGrafanaVersion, err := semver.Parse(metadata.GrafanaVersion) - if err != nil { - return config.Config{}, "", err - } - semverPackageJSONVersion, err := semver.Parse(packageJSONVersion) - if err != nil { - return config.Config{}, "", err - } - // Check if the semver digits of the tag are not equal - if semverGrafanaVersion.FinalizeVersion() != semverPackageJSONVersion.FinalizeVersion() { - return config.Config{}, "", cli.Exit(fmt.Errorf("package.json version and input tag version differ %s != %s.\nPlease update package.json", packageJSONVersion, metadata.GrafanaVersion), 1) - } - } - - cfg.PackageVersion = metadata.GrafanaVersion - return cfg, mode, nil -} diff --git a/pkg/build/frontend/config_test.go b/pkg/build/frontend/config_test.go deleted file mode 100644 index 770df3be7a6..00000000000 --- a/pkg/build/frontend/config_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package frontend - -import ( - "encoding/json" - "flag" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" - - "github.com/grafana/grafana/pkg/build/config" -) - -const ( - jobs = "jobs" - githubToken = "github-token" - buildID = "build-id" -) - -type packageJson struct { - Version string `json:"version"` -} - -type flagObj struct { - name string - value string -} - -var app = cli.NewApp() - -func TestGetConfig(t *testing.T) { - tests := []struct { - ctx *cli.Context - name string - packageJsonVersion string - metadata config.Metadata - wantErr bool - }{ - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "package.json matches tag", - packageJsonVersion: "10.0.0", - metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, - wantErr: false, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "custom tag, package.json doesn't match", - packageJsonVersion: "10.0.0", - metadata: config.Metadata{GrafanaVersion: "10.0.0-abcd123pre", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, - wantErr: false, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "package.json doesn't match tag", - packageJsonVersion: "10.1.0", - metadata: config.Metadata{GrafanaVersion: "10.0.0", ReleaseMode: config.ReleaseMode{Mode: config.TagMode}}, - wantErr: true, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}), nil), - name: "test tag event, check should be skipped", - packageJsonVersion: "10.1.0", - metadata: config.Metadata{GrafanaVersion: "10.1.0-test", ReleaseMode: config.ReleaseMode{Mode: config.TagMode, IsTest: true}}, - wantErr: false, - }, - { - ctx: cli.NewContext(app, setFlags(t, flag.NewFlagSet("flagSet", flag.ContinueOnError), flagObj{name: jobs, value: "2"}, flagObj{name: githubToken, value: "token"}, flagObj{name: buildID, value: "12345"}), nil), - name: "non-tag event", - packageJsonVersion: "10.1.0-pre", - metadata: config.Metadata{GrafanaVersion: "10.1.0-12345pre", ReleaseMode: config.ReleaseMode{Mode: config.PullRequestMode}}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := createTempPackageJson(t, tt.packageJsonVersion) - require.NoError(t, err) - - got, _, err := GetConfig(tt.ctx, tt.metadata) - if !tt.wantErr { - require.Equal(t, got.PackageVersion, tt.metadata.GrafanaVersion) - } - - if tt.wantErr { - require.Equal(t, got.PackageVersion, "") - require.Error(t, err) - } - }) - } -} - -func setFlags(t *testing.T, flagSet *flag.FlagSet, flags ...flagObj) *flag.FlagSet { - t.Helper() - for _, f := range flags { - if f.name != "" { - flagSet.StringVar(&f.name, f.name, f.value, "") - } - } - return flagSet -} - -func createTempPackageJson(t *testing.T, version string) error { - t.Helper() - - data := packageJson{Version: version} - file, _ := json.MarshalIndent(data, "", " ") - - err := os.WriteFile("package.json", file, 0644) - require.NoError(t, err) - - t.Cleanup(func() { - err := os.RemoveAll("package.json") - require.NoError(t, err) - }) - return nil -} diff --git a/pkg/build/fsutil/copy_recursive.go b/pkg/build/fsutil/copy_recursive.go deleted file mode 100644 index daf60a5f5e9..00000000000 --- a/pkg/build/fsutil/copy_recursive.go +++ /dev/null @@ -1,65 +0,0 @@ -package fsutil - -import ( - "fmt" - "os" - "path/filepath" -) - -// CopyRecursive copies files and directories recursively. -func CopyRecursive(src, dst string) error { - sfi, err := os.Stat(src) - if err != nil { - return err - } - if !sfi.IsDir() { - return CopyFile(src, dst) - } - - if _, err := os.Stat(dst); os.IsNotExist(err) { - if err := os.MkdirAll(dst, sfi.Mode()); err != nil { - return fmt.Errorf("failed to create directory %q: %s", dst, err) - } - } - - entries, err := os.ReadDir(src) - if err != nil { - return err - } - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - srcFi, err := os.Stat(srcPath) - if err != nil { - return err - } - - switch srcFi.Mode() & os.ModeType { - case os.ModeDir: - if err := CopyRecursive(srcPath, dstPath); err != nil { - return err - } - case os.ModeSymlink: - link, err := os.Readlink(srcPath) - if err != nil { - return err - } - if err := os.Symlink(link, dstPath); err != nil { - return err - } - default: - if err := CopyFile(srcPath, dstPath); err != nil { - return err - } - } - - if srcFi.Mode()&os.ModeSymlink != 0 { - if err := os.Chmod(dstPath, srcFi.Mode()); err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/build/fsutil/createtemp.go b/pkg/build/fsutil/createtemp.go deleted file mode 100644 index 21720a9a3d8..00000000000 --- a/pkg/build/fsutil/createtemp.go +++ /dev/null @@ -1,43 +0,0 @@ -package fsutil - -import ( - "fmt" - "os" -) - -// CreateTempFile generates a temp filepath, based on the provided suffix. -// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807. -func CreateTempFile(sfx string) (string, error) { - var suffix string - if sfx != "" { - suffix = fmt.Sprintf("*-%s", sfx) - } else { - suffix = sfx - } - f, err := os.CreateTemp("", suffix) - if err != nil { - return "", err - } - if err := f.Close(); err != nil { - return "", err - } - - return f.Name(), nil -} - -// CreateTempDir generates a temp directory, based on the provided suffix. -// A typical generated path looks like /var/folders/abcd/abcdefg/A/1137975807/. -func CreateTempDir(sfx string) (string, error) { - var suffix string - if sfx != "" { - suffix = fmt.Sprintf("*-%s", sfx) - } else { - suffix = sfx - } - dir, err := os.MkdirTemp("", suffix) - if err != nil { - return "", err - } - - return dir, nil -} diff --git a/pkg/build/fsutil/createtemp_test.go b/pkg/build/fsutil/createtemp_test.go deleted file mode 100644 index 640585f4ea5..00000000000 --- a/pkg/build/fsutil/createtemp_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package fsutil - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCreateTempFile(t *testing.T) { - t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807", func(t *testing.T) { - filePath, err := CreateTempFile("") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1) - }) - - t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar", func(t *testing.T) { - filePath, err := CreateTempFile("foobar") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2) - }) -} - -func TestCreateTempDir(t *testing.T) { - t.Run("empty suffix, expects pattern like: /var/folders/abcd/abcdefg/A/1137975807/", func(t *testing.T) { - filePath, err := CreateTempFile("") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 1) - }) - - t.Run("non-empty suffix, expects /var/folders/abcd/abcdefg/A/1137975807-foobar/", func(t *testing.T) { - filePath, err := CreateTempFile("foobar") - require.NoError(t, err) - - pathParts := strings.Split(filePath, "/") - require.Greater(t, len(pathParts), 1) - require.Len(t, strings.Split(pathParts[len(pathParts)-1], "-"), 2) - }) -} diff --git a/pkg/build/gcom/url.go b/pkg/build/gcom/url.go new file mode 100644 index 00000000000..e85c1039796 --- /dev/null +++ b/pkg/build/gcom/url.go @@ -0,0 +1,72 @@ +package gcom + +import ( + "fmt" + "net/url" + "path" + "strings" + + "github.com/grafana/grafana/pkg/build/versions" +) + +func PackageName(grafana, distro, arch, version, ext string, musl bool, raspberryPi bool) string { + v := versions.ParseSemver(version) + + if raspberryPi { + grafana += "-rpi" + } + + versionString := strings.Join([]string{v.Major, v.Minor, v.Patch}, ".") + fmt.Println("Version string:", versionString) + if distro == "deb" { + if v.BuildMetadata != "" { + versionString += "+" + strings.ReplaceAll(v.BuildMetadata, "-", "~") + } + + if v.Prerelease != "" { + versionString += "~" + v.Prerelease + } + + return strings.Join([]string{grafana, versionString, arch}, "_") + "." + ext + } + + if distro == "rhel" { + if v.BuildMetadata != "" { + versionString += "+" + strings.ReplaceAll(v.BuildMetadata, "-", "~") + } + + if v.Prerelease != "" { + versionString += "~" + v.Prerelease + } + + versionString += "-1" + + // Notable difference between our deb naming and our RPM naming: the file ends with `.arch.ext`, not + // `_arch.ext`. + return strings.Join([]string{grafana, versionString}, "-") + "." + arch + "." + ext + } + + if v.Prerelease != "" { + versionString += "-" + v.Prerelease + } + + if v.BuildMetadata != "" { + versionString += "+" + v.BuildMetadata + } + + if musl { + arch += "-musl" + } + + // grafana-enterprise-1.2.3+example-01.linux-amd64.tar.gz + return fmt.Sprintf("%s-%s.%s-%s.%s", grafana, versionString, distro, arch, ext) +} + +func GetURL(baseURL *url.URL, version, grafana, distro, arch, ext string, musl, raspberryPi bool) *url.URL { + packageName := PackageName(grafana, distro, arch, version, ext, musl, raspberryPi) + return &url.URL{ + Host: baseURL.Host, + Scheme: baseURL.Scheme, + Path: path.Join(baseURL.Path, packageName), + } +} diff --git a/pkg/build/gcom/url_test.go b/pkg/build/gcom/url_test.go new file mode 100644 index 00000000000..680a4326fd1 --- /dev/null +++ b/pkg/build/gcom/url_test.go @@ -0,0 +1,367 @@ +package gcom_test + +import ( + "fmt" + "testing" + + "github.com/grafana/grafana/pkg/build/gcom" + "github.com/stretchr/testify/require" +) + +func TestPackageName(t *testing.T) { + type args struct { + Distro string + Arch string + Version string + Ext string + Musl bool + RaspberryPi bool + + Expect string + } + + cases := []args{ + { + RaspberryPi: true, + Distro: "deb", + Arch: "armhf", + Version: "1.2.3", + Ext: "deb", + Expect: "grafana-rpi_1.2.3_armhf.deb", + }, + { + Distro: "deb", + Arch: "arm64", + Version: "1.2.3", + Ext: "deb", + Expect: "grafana_1.2.3_arm64.deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Version: "1.2.3", + Ext: "rpm", + Expect: "grafana-1.2.3-1.aarch64.rpm", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm.sha256", + Version: "1.2.3", + Expect: "grafana-1.2.3-1.aarch64.rpm.sha256", + }, + { + Distro: "rhel", + Ext: "rpm", + Version: "1.2.3", + Arch: "x86_64", + Expect: "grafana-1.2.3-1.x86_64.rpm", + }, + { + Distro: "rhel", + Ext: "rpm.sha256", + Version: "1.2.3", + Arch: "x86_64", + Expect: "grafana-1.2.3-1.x86_64.rpm.sha256", + }, + { + Distro: "darwin", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.darwin-amd64.tar.gz", + }, + { + Distro: "darwin", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.darwin-amd64.tar.gz.sha256", + }, + { + Distro: "darwin", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64-musl.tar.gz", + Musl: true, + }, + { + Distro: "darwin", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64-musl.tar.gz.sha256", + Musl: true, + }, + { + Distro: "darwin", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64.tar.gz", + }, + { + Distro: "darwin", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.darwin-arm64.tar.gz.sha256", + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64-musl.tar.gz", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64-musl.tar.gz.sha256", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64.tar.gz", + }, + { + Distro: "linux", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "amd64", + Expect: "grafana-1.2.3.linux-amd64.tar.gz.sha256", + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64-musl.tar.gz", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz.sha256", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64-musl.tar.gz.sha256", + Musl: true, + }, + { + Distro: "linux", + Ext: "tar.gz", + Version: "1.2.3", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64.tar.gz", + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "arm64", + Expect: "grafana-1.2.3.linux-arm64.tar.gz.sha256", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "linux", + Arch: "armv6", + Expect: "grafana-1.2.3.linux-armv6.tar.gz", + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "armv6", + Expect: "grafana-1.2.3.linux-armv6.tar.gz.sha256", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7-musl.tar.gz", + Musl: true, + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7-musl.tar.gz.sha256", + Musl: true, + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7.tar.gz", + }, + { + Ext: "tar.gz.sha256", + Version: "1.2.3", + Distro: "linux", + Arch: "armv7", + Expect: "grafana-1.2.3.linux-armv7.tar.gz.sha256", + }, + { + Version: "1.2.3", + Arch: "amd64", + Ext: "exe", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.exe", + }, + { + Version: "1.2.3", + Arch: "amd64", + Distro: "windows", + Ext: "exe.sha256", + Expect: "grafana-1.2.3.windows-amd64.exe.sha256", + }, + { + Version: "1.2.3", + Arch: "amd64", + Distro: "windows", + Ext: "msi", + Expect: "grafana-1.2.3.windows-amd64.msi", + }, + { + Version: "1.2.3", + Arch: "amd64", + Distro: "windows", + Ext: "msi.sha256", + Expect: "grafana-1.2.3.windows-amd64.msi.sha256", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.tar.gz", + Arch: "amd64", + }, + { + Version: "1.2.3", + Distro: "windows", + Arch: "amd64", + Ext: "tar.gz.sha256", + Expect: "grafana-1.2.3.windows-amd64.tar.gz.sha256", + }, + { + Version: "1.2.3", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.zip", + Ext: "zip", + Arch: "amd64", + }, + { + Version: "1.2.3", + Distro: "windows", + Expect: "grafana-1.2.3.windows-amd64.zip.sha256", + Ext: "zip.sha256", + Arch: "amd64", + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64-musl.tar.gz", + Musl: true, + }, + { + Version: "1.2.3", + Ext: "tar.gz.sha256", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64-musl.tar.gz.sha256", + Musl: true, + }, + { + Ext: "tar.gz", + Version: "1.2.3", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64.tar.gz", + }, + { + Version: "1.2.3", + Ext: "tar.gz.sha256", + Distro: "windows", + Arch: "arm64", + Expect: "grafana-1.2.3.windows-arm64.tar.gz.sha256", + }, + { + RaspberryPi: true, + Version: "1.2.3", + Ext: "deb", + Arch: "armhf", + Distro: "deb", + Expect: "grafana-rpi_1.2.3_armhf.deb", + }, + { + RaspberryPi: true, + Version: "1.2.3", + Ext: "deb.sha256", + Distro: "deb", + Arch: "armhf", + Expect: "grafana-rpi_1.2.3_armhf.deb.sha256", + }, + { + Version: "1.2.3", + Ext: "deb", + Distro: "deb", + Expect: "grafana_1.2.3_amd64.deb", + Arch: "amd64", + }, + { + Version: "1.2.3", + Ext: "deb.sha256", + Distro: "deb", + Expect: "grafana_1.2.3_amd64.deb.sha256", + Arch: "amd64", + }, + { + Version: "1.2.3", + Ext: "deb", + Arch: "arm64", + Distro: "deb", + Expect: "grafana_1.2.3_arm64.deb", + }, + { + Version: "1.2.3", + Ext: "deb.sha256", + Arch: "arm64", + Distro: "deb", + Expect: "grafana_1.2.3_arm64.deb.sha256", + }, + { + Version: "1.2.3", + Ext: "deb", + Distro: "deb", + Arch: "armhf", + Expect: "grafana_1.2.3_armhf.deb", + }, + { + Version: "1.2.3", + Ext: "deb.sha256", + Arch: "armhf", + Distro: "deb", + Expect: "grafana_1.2.3_armhf.deb.sha256", + }, + } + + for i, v := range cases { + t.Run(fmt.Sprintf("[%d / %d] %s", i+1, len(cases), v.Expect), func(t *testing.T) { + n := gcom.PackageName("grafana", v.Distro, v.Arch, v.Version, v.Ext, v.Musl, v.RaspberryPi) + require.Equal(t, v.Expect, n) + }) + } +} diff --git a/pkg/build/git/git.go b/pkg/build/git/git.go index 9fbcc346e92..2977cf13775 100644 --- a/pkg/build/git/git.go +++ b/pkg/build/git/git.go @@ -4,13 +4,9 @@ import ( "context" "errors" "fmt" - "net/http" "regexp" "github.com/google/go-github/v45/github" - "golang.org/x/oauth2" - - "github.com/grafana/grafana/pkg/build/stringutil" ) const ( @@ -47,19 +43,6 @@ type StatusesService interface { CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) } -// NewGitHubClient creates a new Client using the provided GitHub token if not empty. -func NewGitHubClient(ctx context.Context, token string) *github.Client { - var tc *http.Client - if token != "" { - ts := oauth2.StaticTokenSource(&oauth2.Token{ - AccessToken: token, - }) - tc = oauth2.NewClient(ctx, ts) - } - - return github.NewClient(tc) -} - func PRCheckRegexp() *regexp.Regexp { reBranch, err := regexp.Compile(`^prc-([0-9]+)-([A-Za-z0-9]+)\/(.+)$`) if err != nil { @@ -68,76 +51,3 @@ func PRCheckRegexp() *regexp.Regexp { return reBranch } - -func AddLabelToPR(ctx context.Context, client LabelsService, prID int, newLabel string) error { - // Check existing labels - labels, _, err := client.ListLabelsByIssue(ctx, RepoOwner, OSSRepo, prID, nil) - if err != nil { - return err - } - - duplicate := false - for _, label := range labels { - if *label.Name == newLabel { - duplicate = true - continue - } - - // Delete existing "enterprise-xx" labels - if stringutil.Contains(EnterpriseCheckLabels, *label.Name) { - _, err := client.RemoveLabelForIssue(ctx, RepoOwner, OSSRepo, prID, *label.Name) - if err != nil { - return err - } - } - } - - if duplicate { - return nil - } - - _, _, err = client.AddLabelsToIssue(ctx, RepoOwner, OSSRepo, prID, []string{newLabel}) - if err != nil { - return err - } - - return nil -} - -func DeleteEnterpriseBranch(ctx context.Context, client GitService, branchName string) error { - ref := "heads/" + branchName - if _, err := client.DeleteRef(ctx, RepoOwner, EnterpriseRepo, ref); err != nil { - return err - } - - return nil -} - -// CreateEnterpriseStatus sets the status on a commit for the enterprise build check. -func CreateEnterpriseStatus(ctx context.Context, client StatusesService, sha, link, status string) (*github.RepoStatus, error) { - check, _, err := client.CreateStatus(ctx, RepoOwner, OSSRepo, sha, &github.RepoStatus{ - Context: github.String(EnterpriseCheckName), - Description: github.String(EnterpriseCheckDescription), - TargetURL: github.String(link), - State: github.String(status), - }) - - if err != nil { - return nil, err - } - - return check, nil -} - -func CreateEnterpriseBuildFailedComment(ctx context.Context, client CommentService, link string, prID int) error { - body := fmt.Sprintf("Drone build failed: %s", link) - - _, _, err := client.CreateComment(ctx, RepoOwner, OSSRepo, prID, &github.IssueComment{ - Body: &body, - }) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/build/git/git_checks_test.go b/pkg/build/git/git_checks_test.go deleted file mode 100644 index c9bf0c98d0b..00000000000 --- a/pkg/build/git/git_checks_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package git_test - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-github/v45/github" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/build/git" -) - -type TestChecksService struct { - CreateCheckRunError error -} - -func (s *TestChecksService) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) { - if s.CreateCheckRunError != nil { - return nil, nil, s.CreateCheckRunError - } - - return &github.RepoStatus{ - ID: github.Int64(1), - URL: status.URL, - }, nil, nil -} - -func TestCreateEnterpriseRepoStatus(t *testing.T) { - t.Run("It should create a repo status", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestChecksService{} - link = "http://example.com" - sha = "1234" - ) - - _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") - - require.NoError(t, err) - }) - t.Run("It should return an error if GitHub fails to create the status", func(t *testing.T) { - var ( - ctx = context.Background() - createCheckError = errors.New("create check run error") - client = &TestChecksService{ - CreateCheckRunError: createCheckError, - } - link = "http://example.com" - sha = "1234" - ) - - _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") - require.ErrorIs(t, err, createCheckError) - }) -} diff --git a/pkg/build/git/git_issues_test.go b/pkg/build/git/git_issues_test.go deleted file mode 100644 index dab9fddde77..00000000000 --- a/pkg/build/git/git_issues_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package git_test - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-github/v45/github" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/build/git" -) - -type TestLabelsService struct { - Labels []*github.Label - ListLabelsError error - RemoveLabelError error - AddLabelsError error -} - -func (s *TestLabelsService) ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) { - if s.ListLabelsError != nil { - return nil, nil, s.ListLabelsError - } - - labels := s.Labels - if labels == nil { - labels = []*github.Label{} - } - - return labels, nil, nil -} - -func (s *TestLabelsService) RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) { - if s.RemoveLabelError != nil { - return nil, s.RemoveLabelError - } - - return &github.Response{}, nil -} - -func (s *TestLabelsService) AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) { - if s.AddLabelsError != nil { - return nil, nil, s.AddLabelsError - } - - l := make([]*github.Label, len(labels)) - for i, v := range labels { - l[i] = &github.Label{ - Name: github.String(v), - } - } - - return l, nil, nil -} - -func TestAddLabelToPR(t *testing.T) { - t.Run("It should add a label to a pull request", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestLabelsService{} - pr = 20 - label = "test-label" - ) - - require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) - }) - t.Run("It should not return an error if the label already exists", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestLabelsService{ - Labels: []*github.Label{ - { - Name: github.String("test-label"), - }, - }, - } - pr = 20 - label = "test-label" - ) - - require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) - }) - - t.Run("It should return an error if GitHub returns an error when listing labels", func(t *testing.T) { - var ( - ctx = context.Background() - listLabelsError = errors.New("list labels error") - client = &TestLabelsService{ - ListLabelsError: listLabelsError, - Labels: []*github.Label{}, - } - pr = 20 - label = "test-label" - ) - - require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), listLabelsError) - }) - - t.Run("It should not return an error if there are existing enterprise-check labels.", func(t *testing.T) { - var ( - ctx = context.Background() - client = &TestLabelsService{ - Labels: []*github.Label{ - { - Name: github.String("enterprise-failed"), - }, - }, - } - pr = 20 - label = "test-label" - ) - - require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) - }) - - t.Run("It should return an error if GitHub returns an error when removing existing enterprise-check labels", func(t *testing.T) { - var ( - ctx = context.Background() - removeLabelError = errors.New("remove label error") - client = &TestLabelsService{ - RemoveLabelError: removeLabelError, - Labels: []*github.Label{ - { - Name: github.String("enterprise-failed"), - }, - }, - } - pr = 20 - label = "test-label" - ) - - require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), removeLabelError) - }) -} diff --git a/pkg/build/git/git_test.go b/pkg/build/git/git_test.go deleted file mode 100644 index 3b47ce08dd0..00000000000 --- a/pkg/build/git/git_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package git_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/grafana/grafana/pkg/build/git" -) - -func TestPRCheckRegexp(t *testing.T) { - type match struct { - String string - Commit string - Branch string - PR string - } - - var ( - shouldMatch = []match{ - { - String: "prc-1-a1b2c3d4/branch-name", - Branch: "branch-name", - Commit: "a1b2c3d4", - PR: "1", - }, - { - String: "prc-111-a1b2c3d4/branch/name", - Branch: "branch/name", - Commit: "a1b2c3d4", - PR: "111", - }, - { - String: "prc-102930122-a1b2c3d4/branch-name", - Branch: "branch-name", - Commit: "a1b2c3d4", - PR: "102930122", - }, - } - - shouldNotMatch = []string{"prc-a/branch", "km/test", "test", "prc", "prc/test", "price"} - ) - - regex := git.PRCheckRegexp() - - for _, v := range shouldMatch { - assert.Truef(t, regex.MatchString(v.String), "regex '%s' should match %s", regex.String(), v) - m := regex.FindStringSubmatch(v.String) - assert.Equal(t, m[1], v.PR) - assert.Equal(t, m[2], v.Commit) - assert.Equal(t, m[3], v.Branch) - } - - for _, v := range shouldNotMatch { - assert.False(t, regex.MatchString(v), "regex '%s' should not match %s", regex.String(), v) - } -} diff --git a/pkg/build/go.mod b/pkg/build/go.mod index 2d855a277ba..147fba20c8e 100644 --- a/pkg/build/go.mod +++ b/pkg/build/go.mod @@ -19,7 +19,6 @@ require ( cloud.google.com/go/storage v1.43.0 // @grafana/grafana-backend-group github.com/Masterminds/semver/v3 v3.2.0 // @grafana/grafana-release-guild github.com/aws/aws-sdk-go v1.55.5 // @grafana/aws-datasources - github.com/blang/semver/v4 v4.0.0 // @grafana/grafana-release-guild github.com/docker/docker v27.3.1+incompatible // @grafana/grafana-release-guild github.com/drone/drone-cli v1.6.1 // @grafana/grafana-release-guild github.com/gogo/protobuf v1.3.2 // indirect; @grafana/alerting-backend diff --git a/pkg/build/go.sum b/pkg/build/go.sum index 3e6e7758deb..dabd580b822 100644 --- a/pkg/build/go.sum +++ b/pkg/build/go.sum @@ -39,8 +39,6 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= diff --git a/pkg/build/golangutils/build.go b/pkg/build/golangutils/build.go deleted file mode 100644 index 14da56a7765..00000000000 --- a/pkg/build/golangutils/build.go +++ /dev/null @@ -1,124 +0,0 @@ -package golangutils - -import ( - "context" - "fmt" - "io" - "os/exec" - "strings" - - "github.com/grafana/grafana/pkg/build/config" -) - -type BuildOpts struct { - // Package refers to the path to the `main` package containing `func main` - Package string - - // Output is used as the -o argument in the go build command - Output string - - // Workdir should define some place in the module where the package path resolves. - // Go commands need to be ran inside a the Go module directory. - Workdir string - - GoOS config.OS - GoArch config.Architecture - GoArm string - Go386 string - CC string - LibC string - - CGoEnabled bool - CGoCFlags string - - // LdFlags are joined by a space character and provided to the -ldflags argument. - // A valid element here would be `-X 'main.version=1.0.0'`. - LdFlags []string - - Stdout io.ReadWriter - Stderr io.ReadWriter - Stdin io.ReadWriter - - // ExtraEnv allows consumers to provide extra env args that are not defined above. - // A single element should be formatted using like so: {NAME}={VALUE}. Example: GOOS=linux. - ExtraEnv []string - - // ExtraArgs allows consumers to provide extra arguments that are not defined above. - // Flag names and values should be two separate elements. - // These flags will be appended to the command arguments before the package path in "go build". - ExtraArgs []string -} - -// Env constructs a list of key/value pairs for setting a build command's environment. -// Should we consider using something to unmarshal the struct to env? -func (opts BuildOpts) Env() []string { - env := []string{} - if opts.CGoEnabled { - env = append(env, "CGO_ENABLED=1") - } - - if opts.GoOS != "" { - env = append(env, fmt.Sprintf("GOOS=%s", opts.GoOS)) - } - - if opts.GoArch != "" { - env = append(env, fmt.Sprintf("GOARCH=%s", opts.GoArch)) - } - - if opts.CC != "" { - env = append(env, fmt.Sprintf("CC=%s", opts.CC)) - } - - if opts.CGoCFlags != "" { - env = append(env, fmt.Sprintf("CGO_CFLAGS=%s", opts.CGoCFlags)) - } - - if opts.GoArm != "" { - env = append(env, fmt.Sprintf("GOARM=%s", opts.GoArm)) - } - - if opts.ExtraEnv != nil { - return append(opts.ExtraEnv, env...) - } - - return env -} - -// Args constructs a list of flags and values for use with the exec.Command type when running "go build". -func (opts BuildOpts) Args() []string { - args := []string{} - - if opts.LdFlags != nil { - args = append(args, "-ldflags", strings.Join(opts.LdFlags, " ")) - } - - if opts.Output != "" { - args = append(args, "-o", opts.Output) - } - - if opts.ExtraArgs != nil { - args = append(args, opts.ExtraArgs...) - } - - args = append(args, opts.Package) - - return args -} - -// Build runs the go build process in the current shell given the opts. -// This function will panic if no Stdout/Stderr/Stdin is provided in the opts. -func RunBuild(ctx context.Context, opts BuildOpts) error { - env := opts.Env() - args := append([]string{"build"}, opts.Args()...) - // Ignore gosec G304 as this function is only used in the build process. - //nolint:gosec - cmd := exec.CommandContext(ctx, "go", args...) - cmd.Env = env - - cmd.Stdout = opts.Stdout - cmd.Stderr = opts.Stderr - cmd.Stdin = opts.Stdin - cmd.Dir = opts.Workdir - - return cmd.Run() -} diff --git a/pkg/build/golangutils/doc.go b/pkg/build/golangutils/doc.go deleted file mode 100644 index 2cb33af05e1..00000000000 --- a/pkg/build/golangutils/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package golangutils holds utility functions, wrappers, and types for building Go binaries for Grafana. -package golangutils diff --git a/pkg/build/gpg/gpg.go b/pkg/build/gpg/gpg.go deleted file mode 100644 index 59f96722014..00000000000 --- a/pkg/build/gpg/gpg.go +++ /dev/null @@ -1,73 +0,0 @@ -package gpg - -import ( - "encoding/base64" - "fmt" - "log" - "os" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/fsutil" -) - -// LoadGPGKeys loads GPG key pair and password from the environment and writes them to corresponding files. -// -// The passed config's GPG fields also get updated. Make sure to call RemoveGPGFiles at application exit. -func LoadGPGKeys(cfg *config.Config) error { - var err error - cfg.GPGPrivateKey, err = fsutil.CreateTempFile("priv.key") - if err != nil { - return err - } - cfg.GPGPublicKey, err = fsutil.CreateTempFile("pub.key") - if err != nil { - return err - } - cfg.GPGPassPath, err = fsutil.CreateTempFile("") - if err != nil { - return err - } - - gpgPrivKey := os.Getenv("GPG_PRIV_KEY") - if gpgPrivKey == "" { - return fmt.Errorf("$GPG_PRIV_KEY must be defined") - } - gpgPubKey := os.Getenv("GPG_PUB_KEY") - if gpgPubKey == "" { - return fmt.Errorf("$GPG_PUB_KEY must be defined") - } - gpgPass := os.Getenv("GPG_KEY_PASSWORD") - if gpgPass == "" { - return fmt.Errorf("$GPG_KEY_PASSWORD must be defined") - } - - gpgPrivKeyB, err := base64.StdEncoding.DecodeString(gpgPrivKey) - if err != nil { - return fmt.Errorf("couldn't decode $GPG_PRIV_KEY: %w", err) - } - gpgPubKeyB, err := base64.StdEncoding.DecodeString(gpgPubKey) - if err != nil { - return fmt.Errorf("couldn't decode $GPG_PUB_KEY: %w", err) - } - - if err := os.WriteFile(cfg.GPGPrivateKey, append(gpgPrivKeyB, '\n'), 0400); err != nil { - return fmt.Errorf("failed to write GPG private key file: %w", err) - } - if err := os.WriteFile(cfg.GPGPublicKey, append(gpgPubKeyB, '\n'), 0400); err != nil { - return fmt.Errorf("failed to write GPG public key file: %w", err) - } - if err := os.WriteFile(cfg.GPGPassPath, []byte(gpgPass+"\n"), 0400); err != nil { - return fmt.Errorf("failed to write GPG password file: %w", err) - } - - return nil -} - -// RemoveGPGFiles removes configured GPG files. -func RemoveGPGFiles(cfg config.Config) { - for _, fpath := range []string{cfg.GPGPrivateKey, cfg.GPGPublicKey, cfg.GPGPassPath} { - if err := os.Remove(fpath); err != nil { - log.Printf("failed to remove %q", fpath) - } - } -} diff --git a/pkg/build/gpg/import.go b/pkg/build/gpg/import.go deleted file mode 100644 index 4542a64d8c9..00000000000 --- a/pkg/build/gpg/import.go +++ /dev/null @@ -1,73 +0,0 @@ -package gpg - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/fsutil" -) - -// writeRpmMacros writes ~/.rpmmacros. -func writeRpmMacros(homeDir, gpgPassPath string) error { - fpath := filepath.Join(homeDir, ".rpmmacros") - content := fmt.Sprintf(`%%_signature gpg -%%_gpg_path %s/.gnupg -%%_gpg_name Grafana -%%_gpgbin /usr/bin/gpg -%%__gpg_sign_cmd %%{__gpg} gpg --batch --yes --pinentry-mode loopback --no-armor --passphrase-file %s --no-secmem-warning -u "%%{_gpg_name}" -sbo %%{__signature_filename} %%{__plaintext_filename} -`, homeDir, gpgPassPath) - //nolint:gosec - if err := os.WriteFile(fpath, []byte(content), 0600); err != nil { - return fmt.Errorf("failed to write %q: %w", fpath, err) - } - - return nil -} - -// Import imports the GPG package signing key. -// ~/.rpmmacros also gets written. -func Import(cfg config.Config) error { - exists, err := fsutil.Exists(cfg.GPGPrivateKey) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("GPG private key file doesn't exist: %q", cfg.GPGPrivateKey) - } - - log.Printf("Importing GPG key %q...", cfg.GPGPrivateKey) - // nolint:gosec - cmd := exec.Command("gpg", "--batch", "--yes", "--no-tty", "--allow-secret-key-import", "--import", - cfg.GPGPrivateKey) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to import private key: %s", output) - } - - homeDir, err := os.UserHomeDir() - if err != nil { - return err - } - - if err := writeRpmMacros(homeDir, cfg.GPGPassPath); err != nil { - return err - } - - pubKeysPath := filepath.Join(homeDir, ".rpmdb", "pubkeys") - if err := os.MkdirAll(pubKeysPath, 0700); err != nil { - return fmt.Errorf("failed to make %s: %w", pubKeysPath, err) - } - gpgPub, err := os.ReadFile(cfg.GPGPublicKey) - if err != nil { - return err - } - //nolint:gosec - if err := os.WriteFile(filepath.Join(homeDir, ".rpmdb", "pubkeys", "grafana.key"), gpgPub, 0400); err != nil { - return fmt.Errorf("failed to write pub key to ~/.rpmdb: %w", err) - } - - return nil -} diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go deleted file mode 100644 index 586b2d22839..00000000000 --- a/pkg/build/grafana/build.go +++ /dev/null @@ -1,129 +0,0 @@ -package grafana - -import ( - "context" - "fmt" - "log" - "os" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/cryptoutil" - "github.com/grafana/grafana/pkg/build/golangutils" -) - -var binaries = []string{"grafana", "grafana-server", "grafana-cli"} - -const ( - SuffixEnterprise2 = "-enterprise2" -) - -const ( - ExtensionExe = ".exe" -) - -func GrafanaLDFlags(version string, r config.Revision) []string { - cmd := []string{ - "-w", - fmt.Sprintf("-X main.version=%s", version), - fmt.Sprintf("-X main.commit=%s", r.SHA256), - fmt.Sprintf("-X main.buildstamp=%d", r.Timestamp), - fmt.Sprintf("-X main.buildBranch=%s", r.Branch), - } - - if r.EnterpriseCommit != "" { - cmd = append(cmd, fmt.Sprintf("-X main.enterpriseCommit=%s", r.EnterpriseCommit)) - } - - return cmd -} - -// BinaryFolder returns the path to where the Grafana binary is build given the provided arguments. -func BinaryFolder(edition config.Edition, args BuildArgs) string { - sfx := "" - if edition == config.EditionEnterprise2 { - sfx = SuffixEnterprise2 - } - - arch := string(args.GoArch) - if args.GoArch == config.ArchARM { - arch = string(args.GoArch) + "v" + args.GoArm - } - - format := fmt.Sprintf("%s-%s", args.GoOS, arch) - if args.LibC != "" { - format += fmt.Sprintf("-%s", args.LibC) - } - format += sfx - - if args.GoOS == config.OSWindows { - format += ExtensionExe - } - - return format -} - -func GrafanaDescriptor(opts golangutils.BuildOpts) string { - libcPart := "" - if opts.LibC != "" { - libcPart = fmt.Sprintf("/%s", opts.LibC) - } - arch := string(opts.GoArch) - if opts.GoArch == config.ArchARM { - arch = string(opts.GoArch) + "v" + opts.GoArm - } - - return fmt.Sprintf("%s/%s%s", opts.GoOS, arch, libcPart) -} - -// BuildGrafanaBinary builds a certain binary according to certain parameters. -func BuildGrafanaBinary(ctx context.Context, name, version string, args BuildArgs, edition config.Edition) error { - opts := args.BuildOpts - opts.ExtraEnv = os.Environ() - - revision, err := config.GrafanaRevision(ctx, opts.Workdir) - if err != nil { - return err - } - - folder := BinaryFolder(edition, args) - - if opts.GoOS == config.OSWindows { - name += ExtensionExe - } - - binary := filepath.Join(opts.Workdir, "bin", folder, name) - opts.Output = binary - - if err := os.RemoveAll(binary); err != nil { - return fmt.Errorf("failed to remove %q: %w", binary, err) - } - - if err := os.RemoveAll(binary + ".md5"); err != nil { - return fmt.Errorf("failed to remove %q: %w", binary+".md5", err) - } - - descriptor := GrafanaDescriptor(opts) - - log.Printf("Building %q for %s", binary, descriptor) - - opts.LdFlags = append(args.LdFlags, GrafanaLDFlags(version, revision)...) - - if edition == config.EditionEnterprise2 { - opts.ExtraArgs = []string{"-tags=pro"} - } - - log.Printf("Running command 'go %s'", opts.Args()) - - if err := golangutils.RunBuild(ctx, opts); err != nil { - return err - } - - // Create an MD5 checksum of the binary, to be included in the archive for - // automatic upgrades. - if err := cryptoutil.MD5File(binary); err != nil { - return err - } - - return nil -} diff --git a/pkg/build/grafana/variant.go b/pkg/build/grafana/variant.go index 6ccd4abb8a4..89654722cf2 100644 --- a/pkg/build/grafana/variant.go +++ b/pkg/build/grafana/variant.go @@ -1,160 +1 @@ package grafana - -import ( - "bytes" - "context" - "fmt" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/compilers" - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/golangutils" -) - -// BuildArgs represent the build parameters that define the "go build" behavior of a single variant. -// These arguments are applied as environment variables and arguments to the "go build" command. -type BuildArgs struct { - golangutils.BuildOpts - DebArch config.Architecture - RPMArch config.Architecture -} - -type BuildVariantOpts struct { - Variant config.Variant - Edition config.Edition - - Version string - GrafanaDir string -} - -// BuildVariant builds a certain variant of the grafana-server and grafana-cli binaries sequentially. -func BuildVariant(ctx context.Context, opts BuildVariantOpts) error { - grafanaDir, err := filepath.Abs(opts.GrafanaDir) - if err != nil { - return err - } - - var ( - args = VariantBuildArgs(opts.Variant) - ) - - for _, binary := range binaries { - // Note that for Golang cmd paths we must use the relative path and the Linux file separators (/) even for Windows users. - var ( - pkg = fmt.Sprintf("./pkg/cmd/%s", binary) - stdout = bytes.NewBuffer(nil) - stderr = bytes.NewBuffer(nil) - ) - - args.Workdir = grafanaDir - args.Stdout = stdout - args.Stderr = stderr - args.Package = pkg - - if err := BuildGrafanaBinary(ctx, binary, opts.Version, args, opts.Edition); err != nil { - return fmt.Errorf("failed to build %s for %s: %w\nstdout: %s\nstderr: %s", pkg, opts.Variant, err, stdout.String(), stderr.String()) - } - } - - return nil -} - -var ldFlagsStatic = []string{"-linkmode=external", "-extldflags=-static"} - -var variantArgs = map[config.Variant]BuildArgs{ - config.VariantArmV6: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM, - GoArm: "6", - CC: compilers.ArmV6, - }, - DebArch: config.ArchARMHF, - }, - config.VariantArmV7: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM, - GoArm: "7", - CC: compilers.Armv7, - }, - DebArch: config.ArchARMHF, - RPMArch: config.ArchARMHFP, - }, - config.VariantArmV7Musl: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM, - GoArm: "7", - LibC: config.LibCMusl, - CC: compilers.Armv7Musl, - LdFlags: ldFlagsStatic, - }, - }, - config.VariantArm64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - CGoEnabled: true, - GoArch: config.ArchARM64, - CC: compilers.Arm64, - }, - DebArch: config.ArchARM64, - RPMArch: "aarch64", - }, - config.VariantArm64Musl: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - GoArch: config.ArchARM64, - CGoEnabled: true, - CC: compilers.Arm64Musl, - LibC: config.LibCMusl, - LdFlags: ldFlagsStatic, - }, - }, - config.VariantDarwinAmd64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSDarwin, - CGoEnabled: true, - GoArch: config.ArchAMD64, - CC: compilers.Osx64, - }, - }, - config.VariantWindowsAmd64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSWindows, - GoArch: config.ArchAMD64, - CC: compilers.Win64, - CGoEnabled: true, - CGoCFlags: "-D_WIN32_WINNT=0x0601", - }, - }, - config.VariantLinuxAmd64: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - GoArch: config.ArchAMD64, - CC: compilers.LinuxX64, - }, - DebArch: config.ArchAMD64, - RPMArch: config.ArchAMD64, - }, - config.VariantLinuxAmd64Musl: { - BuildOpts: golangutils.BuildOpts{ - GoOS: config.OSLinux, - GoArch: config.ArchAMD64, - CC: compilers.LinuxX64Musl, - LibC: config.LibCMusl, - LdFlags: ldFlagsStatic, - }, - }, -} - -func VariantBuildArgs(v config.Variant) BuildArgs { - if val, ok := variantArgs[v]; ok { - return val - } - - return BuildArgs{} -} diff --git a/pkg/build/lerna/lerna.go b/pkg/build/lerna/lerna.go index 3bd67a87b44..048c0dd374d 100644 --- a/pkg/build/lerna/lerna.go +++ b/pkg/build/lerna/lerna.go @@ -2,63 +2,13 @@ package lerna import ( "context" - "encoding/json" "fmt" "os" "os/exec" - "path/filepath" - "strings" - "github.com/grafana/grafana/pkg/build/config" "github.com/grafana/grafana/pkg/build/fsutil" ) -// BuildFrontendPackages will bump the version for the package to the latest canary build -// and build the packages so they are ready for being published, used for generating docs etc. -func BuildFrontendPackages(version string, mode config.Edition, grafanaDir string) error { - err := bumpLernaVersion(version, grafanaDir) - if err != nil { - return err - } - cmd := exec.Command("yarn", "run", "packages:build") - cmd.Dir = grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to build %s frontend packages: %s", mode, output) - } - - return nil -} - -func bumpLernaVersion(version string, grafanaDir string) error { - //nolint:gosec - cmd := exec.Command("yarn", "run", "lerna", "version", version, "--exact", "--no-git-tag-version", "--no-push", "--force-publish", "-y") - cmd.Dir = grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to bump version for frontend packages: %s\n%s", err, output) - } - - return nil -} - -func GetLernaVersion(grafanaDir string) (string, error) { - lernaJSONPath := filepath.Join(grafanaDir, "lerna.json") - //nolint:gosec - lernaJSONB, err := os.ReadFile(lernaJSONPath) - if err != nil { - return "", fmt.Errorf("failed to read %q: %w", lernaJSONPath, err) - } - pkgObj := map[string]any{} - if err := json.Unmarshal(lernaJSONB, &pkgObj); err != nil { - return "", fmt.Errorf("failed decoding %q: %w", lernaJSONPath, err) - } - - version := pkgObj["version"].(string) - if version == "" { - return "", fmt.Errorf("failed to read version from %q", lernaJSONPath) - } - return strings.TrimSpace(version), nil -} - func PackFrontendPackages(ctx context.Context, tag, grafanaDir, artifactsDir string) error { exists, err := fsutil.Exists(artifactsDir) if err != nil { diff --git a/pkg/build/packaging/artifacts.go b/pkg/build/packaging/artifacts.go index 36c1e3258e5..1dc60f18a2b 100644 --- a/pkg/build/packaging/artifacts.go +++ b/pkg/build/packaging/artifacts.go @@ -1,9 +1,6 @@ package packaging import ( - "fmt" - "strings" - "github.com/grafana/grafana/pkg/build/config" ) @@ -12,11 +9,18 @@ const MainFolder = "main" const EnterpriseSfx = "-enterprise" const CacheSettings = "Cache-Control:public, max-age=" -type buildArtifact struct { - Os string - Arch string - urlPostfix string - packagePostfix string +type BuildArtifact struct { + // Distro can be "windows", "darwin", "deb", "rhel", or "linux" + Distro string + Arch string + // Ext is the file extension without the "." + Ext string + Musl bool + RaspberryPi bool + + // URL can be set optionally by another process + // Note: check other repos before determining this to be dead code + URL string } type PublishConfig struct { @@ -32,110 +36,91 @@ type PublishConfig struct { SimulateRelease bool } -const rhelOS = "rhel" -const debOS = "deb" - -func (t buildArtifact) GetURL(baseArchiveURL string, cfg PublishConfig) string { - rev := "" - prefix := "-" - if t.Os == debOS { - prefix = "_" - } else if t.Os == rhelOS { - rev = "-1" - } - - version := cfg.Version - verComponents := strings.Split(version, "-") - if len(verComponents) > 2 { - panic(fmt.Sprintf("Version string contains more than one hyphen: %q", version)) - } - - switch t.Os { - case debOS, rhelOS: - if len(verComponents) > 1 { - // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this - // is considered of lower lexical value than the empty character, and this way pre-release versions are - // considered to be of a lower version than the final version (which lacks this suffix). - version = fmt.Sprintf("%s~%s", verComponents[0], verComponents[1]) - } - } - - // https://dl.grafana.com/oss/main/grafana_8.5.0~54094pre_armhf.deb: 404 Not Found - url := fmt.Sprintf("%s%s%s%s%s%s", baseArchiveURL, t.packagePostfix, prefix, version, rev, t.urlPostfix) - return url -} - -var ArtifactConfigs = []buildArtifact{ +var LinuxArtifacts = []BuildArtifact{ { - Os: debOS, - Arch: "arm64", - urlPostfix: "_arm64.deb", + Distro: "linux", + Arch: "arm64", + Ext: "tar.gz", }, { - Os: rhelOS, - Arch: "arm64", - urlPostfix: ".aarch64.rpm", + Distro: "deb", + Arch: "amd64", + Ext: "deb", }, { - Os: "linux", - Arch: "arm64", - urlPostfix: ".linux-arm64.tar.gz", - }, - // https://github.com/golang/go/issues/58425 disabling arm builds until go issue is resolved - // { - // Os: debOS, - // Arch: "armv7", - // urlPostfix: "_armhf.deb", - // }, - // { - // Os: debOS, - // Arch: "armv6", - // packagePostfix: "-rpi", - // urlPostfix: "_armhf.deb", - // }, - // { - // Os: rhelOS, - // Arch: "armv7", - // urlPostfix: ".armhfp.rpm", - // }, - // { - // Os: "linux", - // Arch: "armv6", - // urlPostfix: ".linux-armv6.tar.gz", - // }, - // { - // Os: "linux", - // Arch: "armv7", - // urlPostfix: ".linux-armv7.tar.gz", - // }, - { - Os: "darwin", - Arch: "amd64", - urlPostfix: ".darwin-amd64.tar.gz", + Distro: "rhel", + Arch: "x86_64", + Ext: "rpm", }, { - Os: "deb", - Arch: "amd64", - urlPostfix: "_amd64.deb", - }, - { - Os: rhelOS, - Arch: "amd64", - urlPostfix: ".x86_64.rpm", - }, - { - Os: "linux", - Arch: "amd64", - urlPostfix: ".linux-amd64.tar.gz", - }, - { - Os: "win", - Arch: "amd64", - urlPostfix: ".windows-amd64.zip", - }, - { - Os: "win-installer", - Arch: "amd64", - urlPostfix: ".windows-amd64.msi", + Distro: "linux", + Arch: "amd64", + Ext: "tar.gz", }, } + +var DarwinArtifacts = []BuildArtifact{ + { + Distro: "darwin", + Arch: "amd64", + Ext: "tar.gz", + }, +} + +var WindowsArtifacts = []BuildArtifact{ + { + Distro: "windows", + Arch: "amd64", + Ext: "zip", + }, + { + Distro: "windows", + Arch: "amd64", + Ext: "msi", + }, +} + +var ARMArtifacts = []BuildArtifact{ + { + Distro: "deb", + Arch: "arm64", + Ext: "deb", + }, + { + Distro: "rhel", + Arch: "aarch64", + Ext: "rpm", + }, + { + Distro: "deb", + Arch: "armhf", + Ext: "deb", + RaspberryPi: false, + }, + { + Distro: "deb", + Arch: "armhf", + RaspberryPi: true, + Ext: "deb", + }, + { + Distro: "linux", + Arch: "armv6", + Ext: "tar.gz", + }, + { + Distro: "linux", + Arch: "armv7", + Ext: "tar.gz", + }, +} + +func join(a []BuildArtifact, b ...[]BuildArtifact) []BuildArtifact { + for i := range b { + a = append(a, b[i]...) + } + + return a +} + +var ArtifactConfigs = join(LinuxArtifacts, DarwinArtifacts, WindowsArtifacts, ARMArtifacts) diff --git a/pkg/build/packaging/docs.go b/pkg/build/packaging/docs.go deleted file mode 100644 index a723cb3d346..00000000000 --- a/pkg/build/packaging/docs.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package packaging holds functions and types for creating the tar.gz, deb, and rpm packages of Grafana. -package packaging diff --git a/pkg/build/packaging/errors.go b/pkg/build/packaging/errors.go deleted file mode 100644 index c20b9edbfae..00000000000 --- a/pkg/build/packaging/errors.go +++ /dev/null @@ -1 +0,0 @@ -package packaging diff --git a/pkg/build/packaging/grafana.go b/pkg/build/packaging/grafana.go deleted file mode 100644 index 7022f1bf469..00000000000 --- a/pkg/build/packaging/grafana.go +++ /dev/null @@ -1,1133 +0,0 @@ -package packaging - -import ( - "archive/tar" - "archive/zip" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/grafana" - "github.com/grafana/grafana/pkg/build/plugins" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -var ( - ErrorNoBinaries = errors.New("no binaries found") - ErrorNoDebArch = errors.New("deb architecture not defined") - ErrorNoRPMArch = errors.New("rpm architecture not defined") -) - -const ( - maxAttempts = 3 - enterpriseSfx = "-enterprise" - enterprise2Sfx = "-enterprise2" - DefaultDebDBBucket = "grafana-aptly-db" - DefaultDebRepoBucket = "grafana-repo" - DefaultRPMRepoBucket = "grafana-repo" - DefaultTTLSeconds = "300" -) - -// PackageRegexp returns a regexp for matching packages corresponding to a certain Grafana edition. -func PackageRegexp(edition config.Edition) *regexp.Regexp { - var sfx string - switch edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = "-enterprise" - case config.EditionEnterprise2: - sfx = "-enterprise2" - default: - panic(fmt.Sprintf("unrecognized edition %q", edition)) - } - rePkg, err := regexp.Compile(fmt.Sprintf(`^grafana%s(?:-rpi)?[-_][^-_]+.*$`, sfx)) - if err != nil { - panic(fmt.Sprintf("Failed to compile regexp: %s", err)) - } - - return rePkg -} - -// PackageGrafana packages Grafana for various variants. -func PackageGrafana( - ctx context.Context, - version string, - grafanaDir string, - cfg config.Config, - edition config.Edition, - variants []config.Variant, - shouldSign bool, - p syncutil.WorkerPool, -) error { - if err := packageGrafana(ctx, edition, version, grafanaDir, variants, shouldSign, p); err != nil { - return err - } - - if cfg.SignPackages { - if err := signRPMPackages(edition, cfg, grafanaDir); err != nil { - return err - } - } - - if err := checksumPackages(grafanaDir, edition); err != nil { - return err - } - - return nil -} - -func packageGrafana( - ctx context.Context, - edition config.Edition, - version string, - grafanaDir string, - variants []config.Variant, - shouldSign bool, - p syncutil.WorkerPool, -) error { - distDir := filepath.Join(grafanaDir, "dist") - exists, err := fsutil.Exists(distDir) - if err != nil { - return err - } - if !exists { - log.Printf("directory %s doesn't exist - creating...", distDir) - //nolint - if err := os.MkdirAll(distDir, 0o755); err != nil { - return fmt.Errorf("couldn't create dist: %w", err) - } - } - - var m pluginsManifest - manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json") - //nolint:gosec - manifestB, err := os.ReadFile(manifestPath) - if err != nil { - return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err) - } - if err := json.Unmarshal(manifestB, &m); err != nil { - return err - } - - g, ctx := errutil.GroupWithContext(ctx) - for _, v := range variants { - packageVariant(ctx, v, edition, version, grafanaDir, shouldSign, g, p, m) - } - if err := g.Wait(); err != nil { - return err - } - - return nil -} - -// packageVariant packages Grafana for a certain variant. -func packageVariant( - ctx context.Context, - v config.Variant, - edition config.Edition, - version string, - grafanaDir string, - shouldSign bool, - g *errutil.Group, - p syncutil.WorkerPool, - m pluginsManifest, -) { - p.Schedule(g.Wrap(func() error { - // We've experienced spurious packaging failures, so retry on failure. - i := 0 - for { - if err := realPackageVariant(ctx, v, edition, version, grafanaDir, m, shouldSign); err != nil { - i++ - if i < maxAttempts { - log.Printf("Packaging for variant %s, %s edition failed: %s, trying again", v, edition, err) - continue - } - - return err - } - - break - } - - return nil - })) -} - -// signRPMPackages signs the RPM packages. -func signRPMPackages(edition config.Edition, cfg config.Config, grafanaDir string) error { - log.Printf("Signing %s RPM packages...", edition) - var sfx string - switch edition { - case config.EditionOSS: - case config.EditionEnterprise: - sfx = enterpriseSfx - case config.EditionEnterprise2: - sfx = enterprise2Sfx - default: - panic(fmt.Sprintf("Unrecognized edition %s", edition)) - } - rpms, err := filepath.Glob(filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-*.rpm", sfx))) - if err != nil { - return err - } - - if len(rpms) > 0 { - rpmArgs := append([]string{"--addsign"}, rpms...) - log.Printf("Invoking rpm with args: %+v", rpmArgs) - //nolint:gosec - cmd := exec.Command("rpm", rpmArgs...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to sign RPM packages: %s", output) - } - if err := os.Remove(cfg.GPGPassPath); err != nil { - return fmt.Errorf("failed to remove %q: %w", cfg.GPGPassPath, err) - } - - log.Printf("Verifying %s RPM packages...", edition) - // The output changed between rpm versions - reOutput := regexp.MustCompile("(?:digests signatures OK)|(?:pgp.+OK)") - for _, p := range rpms { - //nolint:gosec - cmd := exec.Command("rpm", "-K", p) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to verify RPM signature: %w", err) - } - - if !reOutput.Match(output) { - return fmt.Errorf("RPM package %q not verified: %s", p, output) - } - } - } - - return nil -} - -// checksumPackages generates package checksums with SHA-256. -func checksumPackages(grafanaDir string, edition config.Edition) error { - log.Printf("Checksumming %s packages...", edition) - distDir := filepath.Join(grafanaDir, "dist") - rePkg := PackageRegexp(edition) - if err := filepath.Walk(distDir, func(fpath string, info os.FileInfo, err error) error { - if err != nil { - var pathErr *os.PathError - if errors.As(err, &pathErr) { - log.Printf("path error in walk function for file %q: %s", pathErr.Path, pathErr.Err.Error()) - return nil - } - return fmt.Errorf("walking through dist folder failed: %w", err) - } - - if info.IsDir() { - return nil - } - - fname := filepath.Base(fpath) - if strings.HasSuffix(fname, ".sha256") || strings.HasSuffix(fname, ".version") || !rePkg.MatchString(fname) { - log.Printf("Ignoring non-package %q", fpath) - return nil - } - - return shaFile(fpath) - }); err != nil { - return fmt.Errorf("checksumming packages in %q failed: %w", distDir, err) - } - - log.Printf("Successfully checksummed %s packages", edition) - return nil -} - -func shaFile(fpath string) error { - //nolint:gosec - fd, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fpath, err) - } - defer func() { - if err := fd.Close(); err != nil { - log.Println(err) - } - }() - - h := sha256.New() - _, err = io.Copy(h, fd) - if err != nil { - return fmt.Errorf("failed to read %q: %w", fpath, err) - } - - //nolint:gosec - out, err := os.Create(fpath + ".sha256") - if err != nil { - return fmt.Errorf("failed to create %q: %w", fpath+".sha256", err) - } - defer func() { - if err := out.Close(); err != nil { - log.Println("failed to close file", out.Name()) - } - }() - - if _, err = fmt.Fprintf(out, "%x\n", h.Sum(nil)); err != nil { - return fmt.Errorf("failed to write %q: %w", out.Name(), err) - } - - return nil -} - -// createPackage creates a Linux package. -func createPackage(srcDir string, options linuxPackageOptions) error { - binary := "grafana" - cliBinary := "grafana-cli" - serverBinary := "grafana-server" - - packageRoot, err := os.MkdirTemp("", "grafana-linux-pack") - if err != nil { - return fmt.Errorf("failed to create temporary directory: %w", err) - } - defer func() { - if err := os.RemoveAll(packageRoot); err != nil { - log.Println(err) - } - }() - - for _, dname := range []string{ - options.homeDir, - options.configDir, - "etc/init.d", - options.etcDefaultPath, - "usr/lib/systemd/system", - "usr/sbin", - } { - dpath := filepath.Join(packageRoot, dname) - //nolint - if err := os.MkdirAll(dpath, 0o755); err != nil { - return fmt.Errorf("failed to make directory %q: %w", dpath, err) - } - } - - if err := fsutil.CopyFile(filepath.Join(options.wrapperFilePath, binary), - filepath.Join(packageRoot, "usr", "sbin", binary)); err != nil { - return err - } - if err := fsutil.CopyFile(filepath.Join(options.wrapperFilePath, cliBinary), - filepath.Join(packageRoot, "usr", "sbin", cliBinary)); err != nil { - return err - } - if err := fsutil.CopyFile(filepath.Join(options.wrapperFilePath, serverBinary), - filepath.Join(packageRoot, "usr", "sbin", serverBinary)); err != nil { - return err - } - if err := fsutil.CopyFile(options.initdScriptSrc, filepath.Join(packageRoot, options.initdScriptFilePath)); err != nil { - return err - } - if err := fsutil.CopyFile(options.defaultFileSrc, filepath.Join(packageRoot, options.etcDefaultFilePath)); err != nil { - return err - } - if err := fsutil.CopyFile(options.systemdFileSrc, filepath.Join(packageRoot, options.systemdServiceFilePath)); err != nil { - return err - } - if err := fsutil.CopyRecursive(srcDir, filepath.Join(packageRoot, options.homeDir)); err != nil { - return err - } - - if err := executeFPM(options, packageRoot, srcDir); err != nil { - return err - } - - return nil -} -func executeFPM(options linuxPackageOptions, packageRoot, srcDir string) error { - name := "grafana" - vendor := "Grafana" - if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 { - vendor += " Enterprise" - if options.edition == config.EditionEnterprise2 { - name += enterprise2Sfx - } else if options.edition == config.EditionEnterprise { - name += enterpriseSfx - } - } - - if options.goArch == config.ArchARM && options.goArm == "6" { - name += "-rpi" - } - - pkgVersion := packageVersion(options) - args := []string{ - "-s", "dir", - "--description", "Grafana", - "-C", packageRoot, - "--url", "https://grafana.com", - "--maintainer", "contact@grafana.com", - "--config-files", options.initdScriptFilePath, - "--config-files", options.etcDefaultFilePath, - "--config-files", options.systemdServiceFilePath, - "--after-install", options.postinstSrc, - "--version", pkgVersion, - "-p", "dist/", - "--name", name, - "--vendor", vendor, - "-a", string(options.packageArch), - } - if options.prermSrc != "" { - args = append(args, "--before-remove", options.prermSrc) - } - if options.edition == config.EditionEnterprise || options.edition == config.EditionEnterprise2 || options.goArch == config.ArchARMv6 { - args = append(args, "--conflicts", "grafana") - } - if options.edition == config.EditionOSS { - args = append(args, "--license", "\"AGPLv3\"") - } - switch options.packageType { - case packageTypeRpm: - args = append(args, "-t", "rpm", "--rpm-posttrans", "packaging/rpm/control/posttrans") - args = append(args, "--rpm-digest", "sha256") - case packageTypeDeb: - args = append(args, "-t", "deb", "--deb-no-default-config-files") - default: - panic(fmt.Sprintf("Unrecognized package type %d", options.packageType)) - } - for _, dep := range options.depends { - args = append(args, "--depends", dep) - } - args = append(args, ".") - - distDir := filepath.Join(options.grafanaDir, "dist") - log.Printf("Generating package in %q (source directory %q)", distDir, srcDir) - - cmdStr := "fpm" - for _, arg := range args { - if strings.Contains(arg, " ") { - arg = fmt.Sprintf("'%s'", arg) - } - cmdStr += fmt.Sprintf(" %s", arg) - } - log.Printf("Creating %s package: %s...", options.packageType, cmdStr) - const rvmPath = "/etc/profile.d/rvm.sh" - exists, err := fsutil.Exists(rvmPath) - if err != nil { - return err - } - if exists { - cmdStr = fmt.Sprintf("source %q && %s", rvmPath, cmdStr) - log.Printf("Sourcing %q before running fpm", rvmPath) - } - //nolint:gosec - cmd := exec.Command("/bin/bash", "-c", cmdStr) - cmd.Dir = options.grafanaDir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to run fpm: %s", output) - } - - return nil -} - -// copyPubDir copies public/ from grafanaDir to tmpDir. -func copyPubDir(grafanaDir, tmpDir string) error { - srcPubDir := filepath.Join(grafanaDir, "public") - tgtPubDir := filepath.Join(tmpDir, "public") - log.Printf("Copying %q to %q...", srcPubDir, tgtPubDir) - if err := fsutil.CopyRecursive(srcPubDir, tgtPubDir); err != nil { - return fmt.Errorf("failed to copy %q to %q: %w", srcPubDir, tgtPubDir, err) - } - - return nil -} - -// copyBinaries copies binaries from grafanaDir into tmpDir. -func copyBinaries(grafanaDir, tmpDir string, args grafana.BuildArgs, edition config.Edition) error { - tgtDir := filepath.Join(tmpDir, "bin") - //nolint - if err := os.MkdirAll(tgtDir, 0o755); err != nil { - return fmt.Errorf("failed to make directory %q: %w", tgtDir, err) - } - - binDir := filepath.Join(grafanaDir, "bin", grafana.BinaryFolder(edition, args)) - - files, err := os.ReadDir(binDir) - if err != nil { - return fmt.Errorf("failed to list files in %q: %w", binDir, err) - } - - if len(files) == 0 { - return fmt.Errorf("%w in %s", ErrorNoBinaries, binDir) - } - - for _, file := range files { - srcPath := filepath.Join(binDir, file.Name()) - tgtPath := filepath.Join(tgtDir, file.Name()) - - if err := fsutil.CopyFile(srcPath, tgtPath); err != nil { - return err - } - } - - return nil -} - -// copyConfFiles copies configuration files from grafanaDir into tmpDir. -func copyConfFiles(grafanaDir, tmpDir string) error { - //nolint:gosec - if err := os.MkdirAll(filepath.Join(tmpDir, "conf"), 0o755); err != nil { - return fmt.Errorf("failed to create dir %q: %w", filepath.Join(tmpDir, "conf"), err) - } - - confDir := filepath.Join(grafanaDir, "conf") - infos, err := os.ReadDir(confDir) - if err != nil { - return fmt.Errorf("failed to list files in %q: %w", confDir, err) - } - for _, info := range infos { - fpath := filepath.Join(confDir, info.Name()) - if err := fsutil.CopyRecursive(fpath, filepath.Join(tmpDir, "conf", info.Name())); err != nil { - return err - } - } - - return nil -} - -// copyPlugins copies plugins from grafanaDir into tmpDir. -func copyPlugins(ctx context.Context, v config.Variant, grafanaDir, tmpDir string, m pluginsManifest, shouldSign bool) error { - log.Printf("Copying plugins for package variant %s...", v) - - variant2Sfx := map[config.Variant]string{ - config.VariantLinuxAmd64: "linux_amd64", - config.VariantDarwinAmd64: "darwin_amd64", - config.VariantWindowsAmd64: "windows_amd64.exe", - } - - tgtDir := filepath.Join(tmpDir, "plugins-bundled") - exists, err := fsutil.Exists(tgtDir) - if err != nil { - return err - } - if !exists { - //nolint:gosec - if err := os.MkdirAll(tgtDir, 0o755); err != nil { - return err - } - } - pluginsDir := filepath.Join(grafanaDir, "plugins-bundled") - - // External plugins. - for _, pm := range m.Plugins { - srcDir := filepath.Join(pluginsDir, fmt.Sprintf("%s-%s", pm.Name, pm.Version)) - dstDir := filepath.Join(tgtDir, fmt.Sprintf("%s-%s", pm.Name, pm.Version)) - log.Printf("Copying external plugin %q to %q...", srcDir, dstDir) - - //nolint:gosec - jsonB, err := os.ReadFile(filepath.Join(srcDir, "plugin.json")) - if err != nil { - return fmt.Errorf("failed to read %q: %w", filepath.Join(srcDir, "plugin.json"), err) - } - var plugJSON map[string]any - if err := json.Unmarshal(jsonB, &plugJSON); err != nil { - return err - } - - plugExe, ok := plugJSON["executable"].(string) - var wantExe string - if ok && strings.TrimSpace(plugExe) != "" { - sfx := variant2Sfx[v] - if sfx == "" { - log.Printf("External plugin %s-%s doesn't have an executable for variant %s - ignoring", - pm.Name, pm.Version, v) - continue - } - - wantExe = fmt.Sprintf("%s_%s", plugExe, sfx) - log.Printf("The external plugin should contain an executable %q", wantExe) - exists, err := fsutil.Exists(filepath.Join(srcDir, wantExe)) - if err != nil { - return err - } - if !exists { - log.Printf("External plugin %s-%s doesn't have an executable of the right format: %q - ignoring", - pm.Name, pm.Version, wantExe) - continue - } - } - - if err := filepath.Walk(srcDir, func(pth string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - log.Printf("Handling %q", pth) - - relPath := strings.TrimPrefix(pth, srcDir) - relPath = strings.TrimPrefix(relPath, "/") - dstPath := filepath.Join(dstDir, relPath) - - if info.IsDir() { - log.Printf("Making directory %q", dstPath) - //nolint:gosec - return os.MkdirAll(dstPath, info.Mode()) - } - - if wantExe != "" { - m, err := regexp.MatchString(fmt.Sprintf(`^%s_[^/]+$`, plugExe), relPath) - if err != nil { - return err - } - if m && relPath != wantExe { - // Ignore other executable variants - log.Printf("Ignoring executable variant %q", pth) - return nil - } - } - - log.Printf("Copying %q to %q", pth, dstPath) - return fsutil.CopyFile(pth, dstPath) - }); err != nil { - return fmt.Errorf("failed to copy external plugin %q to %q: %w", srcDir, dstDir, err) - } - - if shouldSign { - if err := plugins.BuildManifest(ctx, dstDir, true); err != nil { - return fmt.Errorf("failed to generate signed manifest for external plugin %q: %w", dstDir, err) - } - } - } - - return copyInternalPlugins(pluginsDir, tmpDir) -} - -func copyInternalPlugins(pluginsDir, tmpDir string) error { - tgtDir := filepath.Join(tmpDir, "plugins-bundled", "internal") - srcDir := filepath.Join(pluginsDir, "dist") - - exists, err := fsutil.Exists(tgtDir) - if err != nil { - return err - } - if !exists { - //nolint:gosec - if err := os.MkdirAll(tgtDir, 0o755); err != nil { - return err - } - } - - // Copy over internal plugins. - fis, err := os.ReadDir(srcDir) - if err != nil { - return fmt.Errorf("failed to list internal plugins in %q: %w", srcDir, err) - } - for _, fi := range fis { - srcPath := filepath.Join(srcDir, fi.Name()) - if !fi.IsDir() { - log.Printf("Ignoring non-directory %q", srcPath) - continue - } - - dstPath := filepath.Join(tgtDir, fi.Name()) - log.Printf("Copying internal plugin %q to %q...", srcPath, dstPath) - if err := fsutil.CopyRecursive(srcPath, dstPath); err != nil { - return fmt.Errorf("failed to copy %q to %q: %w", srcPath, dstPath, err) - } - } - - return nil -} - -func realPackageVariant(ctx context.Context, v config.Variant, edition config.Edition, version, grafanaDir string, - m pluginsManifest, shouldSign bool) error { - log.Printf("Packaging Grafana %s for %s...", edition, v) - - enableDeb := false - enableRpm := false - switch v { - case config.VariantLinuxAmd64: - enableDeb = true - enableRpm = true - case config.VariantArmV6: - enableDeb = true - case config.VariantArmV7: - enableDeb = true - enableRpm = true - case config.VariantArm64: - enableDeb = true - enableRpm = true - default: - } - - tmpDir, err := os.MkdirTemp("", "") - if err != nil { - return fmt.Errorf("failed to create temporary directory: %w", err) - } - defer func() { - if err := os.RemoveAll(tmpDir); err != nil { - log.Println(err) - } - }() - - args := grafana.VariantBuildArgs(v) - - if err := copyPubDir(grafanaDir, tmpDir); err != nil { - return err - } - if err := copyBinaries(grafanaDir, tmpDir, args, edition); err != nil { - return err - } - if err := copyConfFiles(grafanaDir, tmpDir); err != nil { - return err - } - if err := copyPlugins(ctx, v, grafanaDir, tmpDir, m, shouldSign); err != nil { - return err - } - - if v == config.VariantWindowsAmd64 { - toolsDir := filepath.Join(tmpDir, "tools") - //nolint:gosec - if err := os.MkdirAll(toolsDir, 0o755); err != nil { - return fmt.Errorf("failed to create tools dir %q: %w", toolsDir, err) - } - - if err := fsutil.CopyFile("/usr/local/go/lib/time/zoneinfo.zip", - filepath.Join(tmpDir, "tools", "zoneinfo.zip")); err != nil { - return err - } - } - - if err := os.WriteFile(filepath.Join(tmpDir, "VERSION"), []byte(version), 0664); err != nil { - return fmt.Errorf("failed to write %s/VERSION: %w", tmpDir, err) - } - - if err := createArchive(tmpDir, edition, v, version, grafanaDir); err != nil { - return err - } - - if enableDeb { - if args.DebArch == "" { - return fmt.Errorf("%w for %s", ErrorNoDebArch, v) - } - - if err := createPackage(tmpDir, linuxPackageOptions{ - edition: edition, - version: version, - grafanaDir: grafanaDir, - goArch: args.GoArch, - goArm: args.GoArm, - packageType: packageTypeDeb, - packageArch: args.DebArch, - homeDir: "/usr/share/grafana", - homeBinDir: "/usr/share/grafana/bin", - binPath: "/usr/sbin", - configDir: "/etc/grafana", - etcDefaultPath: "/etc/default", - etcDefaultFilePath: "/etc/default/grafana-server", - initdScriptFilePath: "/etc/init.d/grafana-server", - systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service", - postinstSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "postinst"), - prermSrc: filepath.Join(grafanaDir, "packaging", "deb", "control", "prerm"), - initdScriptSrc: filepath.Join(grafanaDir, "packaging", "deb", "init.d", "grafana-server"), - defaultFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "default", "grafana-server"), - systemdFileSrc: filepath.Join(grafanaDir, "packaging", "deb", "systemd", "grafana-server.service"), - wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"), - depends: []string{"adduser", "libfontconfig1"}, - }); err != nil { - return err - } - } - - if !enableRpm { - return nil - } - - if args.RPMArch == "" { - return fmt.Errorf("%w for %s", ErrorNoRPMArch, v) - } - - if err := createPackage(tmpDir, linuxPackageOptions{ - edition: edition, - version: version, - grafanaDir: grafanaDir, - goArch: args.GoArch, - packageType: packageTypeRpm, - packageArch: args.RPMArch, - homeDir: "/usr/share/grafana", - homeBinDir: "/usr/share/grafana/bin", - binPath: "/usr/sbin", - configDir: "/etc/grafana", - etcDefaultPath: "/etc/sysconfig", - etcDefaultFilePath: "/etc/sysconfig/grafana-server", - initdScriptFilePath: "/etc/init.d/grafana-server", - systemdServiceFilePath: "/usr/lib/systemd/system/grafana-server.service", - postinstSrc: filepath.Join(grafanaDir, "packaging", "rpm", "control", "postinst"), - initdScriptSrc: filepath.Join(grafanaDir, "packaging", "rpm", "init.d", "grafana-server"), - defaultFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "sysconfig", "grafana-server"), - systemdFileSrc: filepath.Join(grafanaDir, "packaging", "rpm", "systemd", "grafana-server.service"), - wrapperFilePath: filepath.Join(grafanaDir, "packaging", "wrappers"), - depends: []string{"/sbin/service", "fontconfig", "freetype"}, - }); err != nil { - return err - } - - return nil -} - -// pluginManifest has details of an external plugin package. -type pluginManifest struct { - Name string `json:"name"` - Version string `json:"version"` - Checksum string `json:"checksum"` -} - -// pluginsManifest represents a manifest of Grafana's external plugins. -type pluginsManifest struct { - Plugins []pluginManifest `json:"plugins"` -} - -// packageVersion converts a Grafana version into the corresponding package version. -func packageVersion(options linuxPackageOptions) string { - verComponents := strings.Split(options.version, "-") - if len(verComponents) > 2 { - panic(fmt.Sprintf("Version string contains more than one hyphen: %q", options.version)) - } - - switch options.packageType { - case packageTypeDeb, packageTypeRpm: - if len(verComponents) > 1 { - // With Debian and RPM packages, it's customary to prefix any pre-release component with a ~, since this - // is considered of lower lexical value than the empty character, and this way pre-release versions are - // considered to be of a lower version than the final version (which lacks this suffix). - return fmt.Sprintf("%s~%s", verComponents[0], verComponents[1]) - } - - return options.version - default: - panic(fmt.Sprintf("Unrecognized package type %s", options.packageType)) - } -} - -type packageType int - -func (pt packageType) String() string { - switch pt { - case packageTypeDeb: - return "Debian" - case packageTypeRpm: - return "RPM" - default: - panic(fmt.Sprintf("Unrecognized package type %d", pt)) - } -} - -const ( - packageTypeDeb packageType = iota - packageTypeRpm -) - -type linuxPackageOptions struct { - edition config.Edition - packageType packageType - version string - grafanaDir string - goArch config.Architecture - goArm string - packageArch config.Architecture - homeDir string - homeBinDir string - binPath string - configDir string - etcDefaultPath string - etcDefaultFilePath string - initdScriptFilePath string - systemdServiceFilePath string - postinstSrc string - prermSrc string - initdScriptSrc string - defaultFileSrc string - systemdFileSrc string - wrapperFilePath string - - depends []string -} - -// createArchive makes a distribution archive. -func createArchive(srcDir string, edition config.Edition, v config.Variant, version, grafanaDir string) error { - distDir := filepath.Join(grafanaDir, "dist") - exists, err := fsutil.Exists(distDir) - if err != nil { - return err - } - if !exists { - log.Printf("directory %s doesn't exist - creating...", distDir) - //nolint:gosec - if err := os.MkdirAll(distDir, 0o755); err != nil { - return fmt.Errorf("couldn't create dist: %w", err) - } - } - sfx := "" - if edition == config.EditionEnterprise2 { - sfx = enterprise2Sfx - } else if edition == config.EditionEnterprise { - sfx = enterpriseSfx - } - if v != config.VariantWindowsAmd64 { - return createTarball(srcDir, version, string(v), sfx, grafanaDir) - } - - return createZip(srcDir, version, string(v), sfx, grafanaDir) -} - -func createZip(srcDir, version, variantStr, sfx, grafanaDir string) error { - fpath := filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-%s.%s.zip", sfx, version, variantStr)) - //nolint:gosec - tgt, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("failed to create %q: %w", fpath, err) - } - defer func() { - if err := tgt.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - log.Println(err) - } - }() - - //nolint:gosec - if err := os.Chmod(fpath, 0664); err != nil { - return fmt.Errorf("failed to set permissions on %q: %w", fpath, err) - } - zipWriter := zip.NewWriter(tgt) - defer func() { - if err := zipWriter.Close(); err != nil { - log.Println(err) - } - }() - - for _, fname := range []string{"LICENSE", "README.md", "NOTICE.md"} { - fpath := filepath.Join(grafanaDir, fname) - fi, err := os.Lstat(fpath) - if err != nil { - return fmt.Errorf("couldn't stat %q: %w", fpath, err) - } - hdr, err := zip.FileInfoHeader(fi) - if err != nil { - return fmt.Errorf("failed to open zip header: %w", err) - } - // Enable compression, as it's disabled by default - hdr.Method = zip.Deflate - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, fname) - w, err := zipWriter.CreateHeader(hdr) - if err != nil { - return fmt.Errorf("failed writing zip header: %w", err) - } - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fname, err) - } - if _, err := io.Copy(w, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing zip entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - } - if err := filepath.Walk(srcDir, func(fpath string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if fpath == srcDir { - return nil - } - - hdr, err := zip.FileInfoHeader(fi) - if err != nil { - return fmt.Errorf("failed to open zip header: %s", err) - } - // Enable compression, as it's disabled by default - hdr.Method = zip.Deflate - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, strings.TrimPrefix(fpath, fmt.Sprintf("%s/", srcDir))) - if fi.IsDir() { - // A trailing slash means it's a directory - if hdr.Name[len(hdr.Name)-1] != '/' { - hdr.Name += "/" - } - } - w, err := zipWriter.CreateHeader(hdr) - if err != nil { - return fmt.Errorf("failed writing zip header: %s", err) - } - if fi.IsDir() { - return nil - } - - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fpath, err) - } - if _, err := io.Copy(w, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing zip entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - return nil - }); err != nil { - return fmt.Errorf("failed to traverse directory %q: %w", srcDir, err) - } - - if err := zipWriter.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - if err := tgt.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - - log.Printf("Successfully created %q", fpath) - return nil -} - -// nolint -func createTarball(srcDir, version, variantStr, sfx, grafanaDir string) error { - fpath := filepath.Join(grafanaDir, "dist", fmt.Sprintf("grafana%s-%s.%s.tar.gz", sfx, version, variantStr)) - //nolint:gosec - tgt, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("failed to create %q: %w", fpath, err) - } - defer func() { - if err := tgt.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - log.Println(err) - } - }() - - //nolint:gosec - if err := os.Chmod(fpath, 0664); err != nil { - return fmt.Errorf("failed to set permissions on %q: %w", fpath, err) - } - gzWriter, err := gzip.NewWriterLevel(tgt, gzip.BestCompression) - if err != nil { - return fmt.Errorf("failed to create gzip writer: %w", err) - } - defer func() { - if err := gzWriter.Close(); err != nil { - log.Println(err) - } - }() - tarWriter := tar.NewWriter(gzWriter) - defer func() { - if err := tarWriter.Close(); err != nil { - log.Println(err) - } - }() - - for _, fname := range []string{"LICENSE", "README.md", "NOTICE.md"} { - fpath := filepath.Join(grafanaDir, fname) - fi, err := os.Lstat(fpath) - if err != nil { - return fmt.Errorf("couldn't stat %q: %w", fpath, err) - } - hdr, err := tar.FileInfoHeader(fi, "") - if err != nil { - return fmt.Errorf("failed getting tar header: %w", err) - } - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, fname) - if err := tarWriter.WriteHeader(hdr); err != nil { - return fmt.Errorf("failed writing tar header: %w", err) - } - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fname, err) - } - if _, err := io.Copy(tarWriter, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing tar entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - } - if err := filepath.Walk(srcDir, func(fpath string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if fpath == srcDir { - return nil - } - - linkTgt := "" - if fi.Mode()&os.ModeSymlink != 0 { - log.Printf("reading link '%s'", fpath) - linkTgt, err = os.Readlink(fpath) - if err != nil { - return err - } - linkTgt = fmt.Sprintf("grafana-%s/%s", version, linkTgt) - } - - hdr, err := tar.FileInfoHeader(fi, linkTgt) - if err != nil { - return fmt.Errorf("failed getting tar header: %w", err) - } - hdr.Name = fmt.Sprintf("grafana-%s/%s", version, strings.TrimPrefix(fpath, fmt.Sprintf("%s/", srcDir))) - if err := tarWriter.WriteHeader(hdr); err != nil { - return fmt.Errorf("failed writing tar header: %w", err) - } - if fi.IsDir() { - return nil - } - - //nolint:gosec - src, err := os.Open(fpath) - if err != nil { - return fmt.Errorf("failed to open %q: %w", fpath, err) - } - if _, err := io.Copy(tarWriter, src); err != nil { - if err := src.Close(); err != nil { - log.Println(err) - } - return fmt.Errorf("failed writing tar entry: %w", err) - } - if err := src.Close(); err != nil { - log.Println(err) - } - - return nil - }); err != nil { - return fmt.Errorf("failed to traverse directory %q: %w", srcDir, err) - } - - if err := tarWriter.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - if err := gzWriter.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - if err := tgt.Close(); err != nil { - return fmt.Errorf("failed writing %q: %w", fpath, err) - } - - st, err := os.Stat(fpath) - if err != nil { - return err - } - perms := st.Mode() & os.ModePerm - log.Printf("Successfully created package %q (permissions: %o)", fpath, perms) - - return nil -} diff --git a/pkg/build/packaging/grafana_test.go b/pkg/build/packaging/grafana_test.go deleted file mode 100644 index 7f45a632718..00000000000 --- a/pkg/build/packaging/grafana_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package packaging_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/packaging" -) - -func TestPackageRegexp(t *testing.T) { - t.Run("It should match enterprise2 packages", func(t *testing.T) { - rgx := packaging.PackageRegexp(config.EditionEnterprise2) - matches := []string{ - "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz", - "grafana-enterprise2-1.2.3-4567pre.linux-amd64.tar.gz.sha256", - } - for _, v := range matches { - assert.Truef(t, rgx.MatchString(v), "'%s' should match regex '%s'", v, rgx.String()) - } - }) -} diff --git a/pkg/build/plugins/build.go b/pkg/build/plugins/build.go deleted file mode 100644 index da9a23cd5d1..00000000000 --- a/pkg/build/plugins/build.go +++ /dev/null @@ -1,66 +0,0 @@ -package plugins - -import ( - "context" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/config" - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/fsutil" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -type PluginSigningMode = int - -// BuildPlugins builds internal plugins. -// The built plugins are placed in plugins-bundled/dist/. -func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.BuildConfig) error { - log.Printf("Building plugins in %q...", grafanaDir) - - root := filepath.Join(grafanaDir, "plugins-bundled", "internal") - fis, err := os.ReadDir(root) - if err != nil { - return err - } - - for i := range fis { - fi := fis[i] - if !fi.IsDir() { - continue - } - - dpath := filepath.Join(root, fi.Name()) - - p.Schedule(g.Wrap(func() error { - log.Printf("Building plugin %q...", dpath) - - cmd := exec.Command("yarn", "build") - cmd.Dir = dpath - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("yarn build failed: %s", output) - } - - dstPath := filepath.Join("plugins-bundled", "dist", fi.Name()) - if err := fsutil.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil { - return err - } - if !verMode.PluginSignature.Sign { - return nil - } - - return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign) - })) - } - - if err := g.Wait(); err != nil { - return err - } - - log.Printf("Built all plug-ins successfully!") - - return nil -} diff --git a/pkg/build/plugins/download.go b/pkg/build/plugins/download.go deleted file mode 100644 index ed8f00bf911..00000000000 --- a/pkg/build/plugins/download.go +++ /dev/null @@ -1,118 +0,0 @@ -package plugins - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - - "github.com/grafana/grafana/pkg/build/errutil" - "github.com/grafana/grafana/pkg/build/syncutil" -) - -// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package. -func logCloseError(closeFunc func() error) { - if err := closeFunc(); err != nil { - log.Println(err) - } -} - -// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package. -func logError(err error) { - if err != nil { - log.Println(err) - } -} - -// pluginManifest has details of an external plugin package. -type pluginManifest struct { - Name string `json:"name"` - Version string `json:"version"` - Checksum string `json:"checksum"` -} - -// pluginsManifest represents a manifest of Grafana's external plugins. -type pluginsManifest struct { - Plugins []pluginManifest `json:"plugins"` -} - -// downloadPlugins downloads Grafana plugins that should be bundled into packages. -// -// The plugin archives are downloaded into /plugins-bundled. -func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error { - g, _ := errutil.GroupWithContext(ctx) - - log.Println("Downloading external plugins...") - - var m pluginsManifest - manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json") - //nolint:gosec - manifestB, err := os.ReadFile(manifestPath) - if err != nil { - return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err) - } - if err := json.Unmarshal(manifestB, &m); err != nil { - return err - } - - for i := range m.Plugins { - pm := m.Plugins[i] - p.Schedule(g.Wrap(func() error { - tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version)) - //nolint:gosec - out, err := os.Create(tgt) - if err != nil { - return err - } - defer logCloseError(out.Close) - - u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name, - pm.Version) - log.Printf("Downloading plugin %q to %q...", u, tgt) - // nolint:gosec - resp, err := http.Get(u) - if err != nil { - return fmt.Errorf("downloading %q failed: %w", u, err) - } - defer logError(resp.Body.Close()) - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode) - } - - if _, err := io.Copy(out, resp.Body); err != nil { - return fmt.Errorf("downloading %q failed: %w", u, err) - } - if err := out.Close(); err != nil { - return fmt.Errorf("downloading %q failed: %w", u, err) - } - - //nolint:gosec - fd, err := os.Open(tgt) - if err != nil { - return err - } - defer logCloseError(fd.Close) - - h := sha256.New() - if _, err := io.Copy(h, fd); err != nil { - return err - } - - chksum := hex.EncodeToString(h.Sum(nil)) - if chksum != pm.Checksum { - return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum) - } - - return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled")) - })) - } - - return g.Wait() -} diff --git a/pkg/build/plugins/manifest.go b/pkg/build/plugins/manifest.go deleted file mode 100644 index 359e8ad77bc..00000000000 --- a/pkg/build/plugins/manifest.go +++ /dev/null @@ -1,204 +0,0 @@ -package plugins - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" -) - -type manifest struct { - Plugin string `json:"plugin"` - Version string `json:"version"` - Files map[string]string `json:"files"` -} - -func getManifest(dpath string, chksums map[string]string) (manifest, error) { - m := manifest{} - - type pluginInfo struct { - Version string `json:"version"` - } - - type plugin struct { - ID string `json:"id"` - Info pluginInfo `json:"info"` - } - - //nolint:gosec - f, err := os.Open(filepath.Join(dpath, "plugin.json")) - if err != nil { - return m, err - } - decoder := json.NewDecoder(f) - var p plugin - if err := decoder.Decode(&p); err != nil { - return m, err - } - - if p.ID == "" { - return m, fmt.Errorf("plugin.json doesn't define id") - } - if p.Info.Version == "" { - return m, fmt.Errorf("plugin.json doesn't define info.version") - } - - return manifest{ - Plugin: p.ID, - Version: p.Info.Version, - Files: chksums, - }, nil -} - -// BuildManifest requests a plugin's signed manifest file fromt he Grafana API. -// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires -// an admin API key. -func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error { - log.Printf("Building manifest for plug-in at %q", dpath) - - apiKey := os.Getenv("GRAFANA_API_KEY") - if apiKey == "" { - return fmt.Errorf("GRAFANA_API_KEY must be set") - } - - manifestPath := filepath.Join(dpath, "MANIFEST.txt") - chksums, err := getChksums(dpath, manifestPath) - if err != nil { - return err - } - m, err := getManifest(dpath, chksums) - if err != nil { - return err - } - - b := bytes.NewBuffer(nil) - encoder := json.NewEncoder(b) - if err := encoder.Encode(&m); err != nil { - return err - } - jsonB := b.Bytes() - u := "https://grafana.com/api/plugins/ci/sign" - if !signingAdmin { - u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin) - } - log.Printf("Requesting signed manifest from Grafana API...") - req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB)) - if err != nil { - return err - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - req.Header.Add("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err) - } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Println("failed to close response body, err: %w", err) - } - }() - if resp.StatusCode != 200 { - msg, err := io.ReadAll(resp.Body) - if err != nil { - log.Printf("Failed to read response body: %s", err) - msg = []byte("") - } - return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg)) - } - - log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath) - //nolint:gosec - f, err := os.Create(manifestPath) - if err != nil { - return fmt.Errorf("failed to create %s: %w", manifestPath, err) - } - defer func() { - if err := f.Close(); err != nil { - log.Println("failed to close file, err: %w", err) - } - }() - if _, err := io.Copy(f, resp.Body); err != nil { - return fmt.Errorf("failed to write %s: %w", manifestPath, err) - } - if err := f.Close(); err != nil { - return fmt.Errorf("failed to write %s: %w", manifestPath, err) - } - - return nil -} - -func getChksums(dpath, manifestPath string) (map[string]string, error) { - manifestPath = filepath.Clean(manifestPath) - - chksums := map[string]string{} - if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - - if fi.IsDir() { - return nil - } - - path = filepath.Clean(path) - - // Handle symbolic links - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - finalPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - - log.Printf("Handling symlink %q, pointing to %q", path, finalPath) - - info, err := os.Stat(finalPath) - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - if _, err := filepath.Rel(dpath, finalPath); err != nil { - return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath) - } - - if finalPath == manifestPath { - return nil - } - } - - if path == manifestPath { - return nil - } - - h := sha256.New() - //nolint:gosec - f, err := os.Open(path) - if err != nil { - return err - } - defer logCloseError(f.Close) - if _, err := io.Copy(h, f); err != nil { - return err - } - - relPath, err := filepath.Rel(dpath, path) - if err != nil { - return err - } - chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil)) - - return nil - }); err != nil { - return nil, err - } - - return chksums, nil -} diff --git a/pkg/build/plugins/zip.go b/pkg/build/plugins/zip.go deleted file mode 100644 index 73f8e8d82f6..00000000000 --- a/pkg/build/plugins/zip.go +++ /dev/null @@ -1,64 +0,0 @@ -package plugins - -import ( - "archive/zip" - "io" - "log" - "os" - "path/filepath" -) - -// Unzip unzips a plugin. -func Unzip(fpath, tgtDir string) error { - log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir) - - r, err := zip.OpenReader(fpath) - if err != nil { - return err - } - defer logCloseError(r.Close) - - // Closure to address file descriptors issue with all the deferred .Close() methods - extractAndWriteFile := func(f *zip.File) error { - log.Printf("Extracting zip member %q...", f.Name) - - rc, err := f.Open() - if err != nil { - return err - } - defer logCloseError(rc.Close) - - //nolint:gosec - dstPath := filepath.Join(tgtDir, f.Name) - - if f.FileInfo().IsDir() { - return os.MkdirAll(dstPath, f.Mode()) - } - - if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil { - return err - } - - //nolint:gosec - fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - defer logCloseError(fd.Close) - - // nolint:gosec - if _, err := io.Copy(fd, rc); err != nil { - return err - } - - return fd.Close() - } - - for _, f := range r.File { - if err := extractAndWriteFile(f); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/build/stringutil/contains.go b/pkg/build/stringutil/contains.go deleted file mode 100644 index b53efe70759..00000000000 --- a/pkg/build/stringutil/contains.go +++ /dev/null @@ -1,10 +0,0 @@ -package stringutil - -func Contains(arr []string, s string) bool { - for _, e := range arr { - if e == s { - return true - } - } - return false -} diff --git a/pkg/build/syncutil/pool.go b/pkg/build/syncutil/pool.go deleted file mode 100644 index 6034059d2bf..00000000000 --- a/pkg/build/syncutil/pool.go +++ /dev/null @@ -1,43 +0,0 @@ -package syncutil - -import ( - "log" - "runtime" -) - -func worker(jobs chan func()) { - for j := range jobs { - j() - } -} - -// WorkerPool represents a concurrent worker pool. -type WorkerPool struct { - NumWorkers int - jobs chan func() -} - -// NewWorkerPool constructs a new WorkerPool. -func NewWorkerPool(numWorkers int) WorkerPool { - if numWorkers <= 0 { - numWorkers = runtime.NumCPU() - } - log.Printf("Creating worker pool with %d workers", numWorkers) - jobs := make(chan func(), 100) - for i := 0; i < numWorkers; i++ { - go worker(jobs) - } - return WorkerPool{ - NumWorkers: numWorkers, - jobs: jobs, - } -} - -// Schedule schedules a job to be executed by a worker in the pool. -func (p WorkerPool) Schedule(job func()) { - p.jobs <- job -} - -func (p WorkerPool) Close() { - close(p.jobs) -} diff --git a/pkg/build/validation/validation.go b/pkg/build/validation/validation.go index 17951261114..bc01ed189b7 100644 --- a/pkg/build/validation/validation.go +++ b/pkg/build/validation/validation.go @@ -1,9 +1,5 @@ package validation -import ( - "context" -) - type ArtifactType int const ( @@ -15,13 +11,3 @@ type Artifact struct { Type ArtifactType URL string } - -// ReleaseArtifacts generates a list of release artifacts -func ReleaseArtifacts(version string) ([]Artifact, error) { - return nil, nil -} - -// VerifyRelease tests that a that, given the information, a release will completed wholly and successfully. -func VerifyRelease(ctx context.Context, version string) (bool, error) { - return false, nil -} diff --git a/pkg/build/versions/parse.go b/pkg/build/versions/parse.go new file mode 100644 index 00000000000..0a38af4329c --- /dev/null +++ b/pkg/build/versions/parse.go @@ -0,0 +1,31 @@ +package versions + +import "regexp" + +var semverRegex = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + +type Semver struct { + Major string + Minor string + Patch string + Prerelease string + BuildMetadata string +} + +func ParseSemver(version string) Semver { + matches := semverRegex.FindStringSubmatch(version) + results := make(map[string]string) + for i, name := range semverRegex.SubexpNames() { + if i != 0 && name != "" { + results[name] = matches[i] + } + } + + return Semver{ + Major: results["major"], + Minor: results["minor"], + Patch: results["patch"], + Prerelease: results["prerelease"], + BuildMetadata: results["buildmetadata"], + } +} diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go index c40dc894ee9..143e30756ca 100644 --- a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go @@ -42,6 +42,8 @@ func New(cfg *setting.Cfg, validator validations.PluginRequestValidator, tracer middlewares = append(middlewares, GrafanaRequestIDHeaderMiddleware(cfg, logger)) } + middlewares = append(middlewares, sdkhttpclient.ErrorSourceMiddleware()) + // SigV4 signing should be performed after all headers are added if cfg.SigV4AuthEnabled { authSettings := awsds.AuthSettings{ diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go index 652372a0de4..84279cca2e4 100644 --- a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go @@ -27,7 +27,7 @@ func TestHTTPClientProvider(t *testing.T) { _ = New(&setting.Cfg{SigV4AuthEnabled: false}, &validations.OSSPluginRequestValidator{}, tracer) require.Len(t, providerOpts, 1) o := providerOpts[0] - require.Len(t, o.Middlewares, 8) + require.Len(t, o.Middlewares, 9) require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.ContextualMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) @@ -35,6 +35,8 @@ func TestHTTPClientProvider(t *testing.T) { require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.ResponseLimitMiddlewareName, o.Middlewares[6].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, HostRedirectValidationMiddlewareName, o.Middlewares[7].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.ErrorSourceMiddlewareName, o.Middlewares[8].(sdkhttpclient.MiddlewareName).MiddlewareName()) }) t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) { @@ -51,7 +53,7 @@ func TestHTTPClientProvider(t *testing.T) { _ = New(&setting.Cfg{SigV4AuthEnabled: true}, &validations.OSSPluginRequestValidator{}, tracer) require.Len(t, providerOpts, 1) o := providerOpts[0] - require.Len(t, o.Middlewares, 9) + require.Len(t, o.Middlewares, 10) require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.ContextualMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) @@ -59,7 +61,9 @@ func TestHTTPClientProvider(t *testing.T) { require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.ResponseLimitMiddlewareName, o.Middlewares[6].(sdkhttpclient.MiddlewareName).MiddlewareName()) - require.Equal(t, awssdk.SigV4MiddlewareName, o.Middlewares[8].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, HostRedirectValidationMiddlewareName, o.Middlewares[7].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.ErrorSourceMiddlewareName, o.Middlewares[8].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, awssdk.SigV4MiddlewareName, o.Middlewares[9].(sdkhttpclient.MiddlewareName).MiddlewareName()) }) t.Run("When creating new provider and http logging is enabled for one plugin, it should apply expected middleware", func(t *testing.T) { @@ -76,7 +80,7 @@ func TestHTTPClientProvider(t *testing.T) { _ = New(&setting.Cfg{PluginSettings: setting.PluginSettings{"example": {"har_log_enabled": "true"}}}, &validations.OSSPluginRequestValidator{}, tracer) require.Len(t, providerOpts, 1) o := providerOpts[0] - require.Len(t, o.Middlewares, 9) + require.Len(t, o.Middlewares, 10) require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, sdkhttpclient.ContextualMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) @@ -86,5 +90,6 @@ func TestHTTPClientProvider(t *testing.T) { require.Equal(t, sdkhttpclient.ResponseLimitMiddlewareName, o.Middlewares[6].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, HostRedirectValidationMiddlewareName, o.Middlewares[7].(sdkhttpclient.MiddlewareName).MiddlewareName()) require.Equal(t, HTTPLoggerMiddlewareName, o.Middlewares[8].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.ErrorSourceMiddlewareName, o.Middlewares[9].(sdkhttpclient.MiddlewareName).MiddlewareName()) }) } diff --git a/pkg/plugins/manager/installer.go b/pkg/plugins/manager/installer.go index b1db5662c51..227d8a35450 100644 --- a/pkg/plugins/manager/installer.go +++ b/pkg/plugins/manager/installer.go @@ -75,6 +75,11 @@ func (m *PluginInstaller) Add(ctx context.Context, pluginID, version string, opt err = m.Add(ctx, dep.ID, dep.Version, opts) if err != nil { + var dupeErr plugins.DuplicateError + if errors.As(err, &dupeErr) { + m.log.Info("Dependency already installed", "pluginId", dep.ID) + continue + } return fmt.Errorf("%v: %w", fmt.Sprintf("failed to download plugin %s from repository", dep.ID), err) } } diff --git a/pkg/plugins/manager/installer_test.go b/pkg/plugins/manager/installer_test.go index f90c3ba11c3..bb4564c7d25 100644 --- a/pkg/plugins/manager/installer_test.go +++ b/pkg/plugins/manager/installer_test.go @@ -279,6 +279,50 @@ func TestPluginManager_Add_Remove(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{p2Zip, p1Zip}, loadedPaths) }) + + t.Run("Plugin can successfully install even if dependency plugin is already installed", func(t *testing.T) { + const pluginDependencyID = "test-plugin-dependency" + reg := &fakes.FakePluginRegistry{ + Store: map[string]*plugins.Plugin{ + pluginDependencyID: createPlugin(t, pluginDependencyID, plugins.ClassExternal, false, false), + }, + } + + var loadedPaths []string + loader := &fakes.FakeLoader{ + LoadFunc: func(ctx context.Context, src plugins.PluginSource) ([]*plugins.Plugin, error) { + loadedPaths = append(loadedPaths, src.PluginURIs(ctx)...) + return []*plugins.Plugin{}, nil + }, + } + + pluginRepo := &fakes.FakePluginRepo{ + GetPluginArchiveFunc: func(_ context.Context, id, version string, _ repo.CompatOpts) (*repo.PluginArchive, error) { + return &repo.PluginArchive{File: &zip.ReadCloser{Reader: zip.Reader{File: []*zip.File{{ + FileHeader: zip.FileHeader{Name: fmt.Sprintf("%s.zip", id)}, + }}}}}, nil + }, + } + + fs := &fakes.FakePluginStorage{ + ExtractFunc: func(_ context.Context, id string, _ storage.DirNameGeneratorFunc, z *zip.ReadCloser) (*storage.ExtractedPluginArchive, error) { + switch id { + case testPluginID: + return &storage.ExtractedPluginArchive{ + Dependencies: []*storage.Dependency{{ID: pluginDependencyID}}, + Path: "test-plugin.zip", + }, nil + default: + return nil, fmt.Errorf("unknown plugin %s", id) + } + }, + } + + inst := New(reg, loader, pluginRepo, fs, storage.SimpleDirNameGeneratorFunc, &fakes.FakeAuthService{}) + err := inst.Add(context.Background(), testPluginID, "", testCompatOpts()) + require.NoError(t, err) + require.Equal(t, []string{"test-plugin.zip"}, loadedPaths) + }) } func createPlugin(t *testing.T, pluginID string, class plugins.Class, managed, backend bool, cbs ...func(*plugins.Plugin)) *plugins.Plugin { diff --git a/pkg/plugins/pluginrequestmeta/plugin_request_meta.go b/pkg/plugins/pluginrequestmeta/plugin_request_meta.go deleted file mode 100644 index 1a189bf33fb..00000000000 --- a/pkg/plugins/pluginrequestmeta/plugin_request_meta.go +++ /dev/null @@ -1,59 +0,0 @@ -package pluginrequestmeta - -import ( - "context" - "errors" - - "github.com/grafana/grafana-plugin-sdk-go/backend" -) - -// StatusSource is an enum-like string value representing the source of a -// plugin query data response status code -type StatusSource string - -const ( - StatusSourcePlugin StatusSource = "plugin" - StatusSourceDownstream StatusSource = "downstream" -) - -// DefaultStatusSource is the default StatusSource that should be used when it is not explicitly set by the plugin. -const DefaultStatusSource StatusSource = StatusSourcePlugin - -type statusSourceCtxKey struct{} - -// StatusSourceFromContext returns the plugin request status source stored in the context. -// If no plugin request status source is stored in the context, [DefaultStatusSource] is returned. -func StatusSourceFromContext(ctx context.Context) StatusSource { - value, ok := ctx.Value(statusSourceCtxKey{}).(*StatusSource) - if ok { - return *value - } - return DefaultStatusSource -} - -// WithStatusSource sets the plugin request status source for the context. -func WithStatusSource(ctx context.Context, s StatusSource) context.Context { - return context.WithValue(ctx, statusSourceCtxKey{}, &s) -} - -// WithDownstreamStatusSource mutates the provided context by setting the plugin request status source to -// StatusSourceDownstream. If the provided context does not have a plugin request status source, the context -// will not be mutated. This means that [WithStatusSource] has to be called before this function. -func WithDownstreamStatusSource(ctx context.Context) error { - v, ok := ctx.Value(statusSourceCtxKey{}).(*StatusSource) - if !ok { - return errors.New("the provided context does not have a plugin request status source") - } - *v = StatusSourceDownstream - return nil -} - -// StatusSourceFromPluginErrorSource takes an error source returned by a plugin and returns the corresponding -// StatusSource. If the provided value is a zero-value (i.e.: the plugin did not set it), the function returns -// DefaultStatusSource. -func StatusSourceFromPluginErrorSource(pluginErrorSource backend.ErrorSource) StatusSource { - if pluginErrorSource == "" { - return DefaultStatusSource - } - return StatusSource(pluginErrorSource) -} diff --git a/pkg/plugins/pluginrequestmeta/plugin_request_meta_test.go b/pkg/plugins/pluginrequestmeta/plugin_request_meta_test.go deleted file mode 100644 index ffec48c1e30..00000000000 --- a/pkg/plugins/pluginrequestmeta/plugin_request_meta_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package pluginrequestmeta - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestStatusSource(t *testing.T) { - t.Run("WithStatusSource", func(t *testing.T) { - ctx := context.Background() - ss := StatusSourceFromContext(ctx) - require.Equal(t, StatusSourcePlugin, ss) - - ctx = WithStatusSource(ctx, StatusSourceDownstream) - ss = StatusSourceFromContext(ctx) - require.Equal(t, StatusSourceDownstream, ss) - }) - - t.Run("WithDownstreamStatusSource", func(t *testing.T) { - t.Run("Returns error if no status source is set", func(t *testing.T) { - ctx := context.Background() - err := WithDownstreamStatusSource(ctx) - require.Error(t, err) - require.Equal(t, StatusSourcePlugin, StatusSourceFromContext(ctx)) - }) - - t.Run("Should mutate context if status source is set", func(t *testing.T) { - ctx := WithStatusSource(context.Background(), StatusSourcePlugin) - err := WithDownstreamStatusSource(ctx) - require.NoError(t, err) - require.Equal(t, StatusSourceDownstream, StatusSourceFromContext(ctx)) - }) - }) - - t.Run("StatusSourceFromContext", func(t *testing.T) { - t.Run("Background returns StatusSourcePlugin", func(t *testing.T) { - ctx := context.Background() - ss := StatusSourceFromContext(ctx) - require.Equal(t, StatusSourcePlugin, ss) - }) - - t.Run("Context with status source returns the set status source", func(t *testing.T) { - ctx := WithStatusSource(context.Background(), StatusSourcePlugin) - ss := StatusSourceFromContext(ctx) - require.Equal(t, StatusSourcePlugin, ss) - }) - }) -} diff --git a/pkg/promlib/go.mod b/pkg/promlib/go.mod index 51e5bc9cc60..538022bba05 100644 --- a/pkg/promlib/go.mod +++ b/pkg/promlib/go.mod @@ -4,7 +4,7 @@ go 1.23.1 require ( github.com/grafana/dskit v0.0.0-20240805174438-dfa83b4ed2d3 - github.com/grafana/grafana-plugin-sdk-go v0.251.0 + github.com/grafana/grafana-plugin-sdk-go v0.253.0 github.com/json-iterator/go v1.1.12 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.20.4 diff --git a/pkg/promlib/go.sum b/pkg/promlib/go.sum index af656a4d453..ee2510d54d8 100644 --- a/pkg/promlib/go.sum +++ b/pkg/promlib/go.sum @@ -98,8 +98,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grafana/dskit v0.0.0-20240805174438-dfa83b4ed2d3 h1:as4PmrFoYI1byS5JjsgPC7uSGTMh+SgS0ePv6hOyDGU= github.com/grafana/dskit v0.0.0-20240805174438-dfa83b4ed2d3/go.mod h1:lcjGB6SuaZ2o44A9nD6p/tR4QXSPbzViRY520Gy6pTQ= -github.com/grafana/grafana-plugin-sdk-go v0.251.0 h1:gnOtxrC/1rqFvpSbQYyoZqkr47oWDlz4Q2L6Ozmsi3w= -github.com/grafana/grafana-plugin-sdk-go v0.251.0/go.mod h1:gCGN9kHY3KeX4qyni3+Kead38Q+85pYOrsDcxZp6AIk= +github.com/grafana/grafana-plugin-sdk-go v0.253.0 h1:KaCrqqsDgVIoT8hwvwuUMKV7QbHVlvRoFN5+U2rOXR8= +github.com/grafana/grafana-plugin-sdk-go v0.253.0/go.mod h1:gCGN9kHY3KeX4qyni3+Kead38Q+85pYOrsDcxZp6AIk= github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= diff --git a/pkg/registry/apis/alerting/notifications/register.go b/pkg/registry/apis/alerting/notifications/register.go index f073a3c7d93..72a0b0f3c16 100644 --- a/pkg/registry/apis/alerting/notifications/register.go +++ b/pkg/registry/apis/alerting/notifications/register.go @@ -7,14 +7,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" notificationsModels "github.com/grafana/grafana/pkg/apis/alerting_notifications/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" receiver "github.com/grafana/grafana/pkg/registry/apis/alerting/notifications/receiver" "github.com/grafana/grafana/pkg/registry/apis/alerting/notifications/template_group" timeInterval "github.com/grafana/grafana/pkg/registry/apis/alerting/notifications/timeinterval" @@ -71,7 +69,11 @@ func (t *NotificationsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(notificationsModels.SchemeGroupVersion) } -func (t *NotificationsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, dualWriteBuilder grafanarest.DualWriteBuilder) error { +func (t *NotificationsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { + scheme := opts.Scheme + optsGetter := opts.OptsGetter + dualWriteBuilder := opts.DualWriteBuilder + intervals, err := timeInterval.NewStorage(t.ng.Api.MuteTimings, t.namespacer, scheme, optsGetter, dualWriteBuilder) if err != nil { return fmt.Errorf("failed to initialize time-interval storage: %w", err) diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index ae57adfe147..d5ff691180a 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -5,7 +5,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" @@ -57,8 +56,8 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracing *tracing.TracingService, unified resource.ResourceClient, ) *DashboardsAPIBuilder { - if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { - return nil // skip registration unless opting into experimental apis + if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) && !features.IsEnabledGlobally(featuremgmt.FlagKubernetesDashboardsAPI) { + return nil // skip registration unless opting into experimental apis or dashboards in the k8s api } softDelete := features.IsEnabledGlobally(featuremgmt.FlagDashboardRestore) @@ -124,7 +123,11 @@ func (b *DashboardsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(resourceInfo.GroupVersion()) } -func (b *DashboardsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, dualWriteBuilder grafanarest.DualWriteBuilder) error { +func (b *DashboardsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { + scheme := opts.Scheme + optsGetter := opts.OptsGetter + dualWriteBuilder := opts.DualWriteBuilder + dash := b.legacy.resource legacyStore, err := b.legacy.newStore(scheme, optsGetter) if err != nil { diff --git a/pkg/registry/apis/dashboardsnapshot/register.go b/pkg/registry/apis/dashboardsnapshot/register.go index a8bc461a3c0..08df1d7b691 100644 --- a/pkg/registry/apis/dashboardsnapshot/register.go +++ b/pkg/registry/apis/dashboardsnapshot/register.go @@ -12,7 +12,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" @@ -22,7 +21,6 @@ import ( "github.com/grafana/authlib/claims" "github.com/grafana/grafana/pkg/apimachinery/identity" dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/apiserver/builder" @@ -119,7 +117,7 @@ func (b *SnapshotsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(gv) } -func (b *SnapshotsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ *runtime.Scheme, _ generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *SnapshotsAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ builder.APIGroupOptions) error { storage := map[string]rest.Storage{} legacyStore := &legacyStorage{ diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index d7a3fd963ea..bac6da1f8d9 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -5,11 +5,21 @@ import ( "encoding/json" "fmt" + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/registry/rest" + genericapiserver "k8s.io/apiserver/pkg/server" + openapi "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/spec3" + "k8s.io/utils/strings/slices" + "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana/pkg/apimachinery/utils" datasource "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/promlib/models" @@ -19,17 +29,6 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource/kinds" - "github.com/prometheus/client_golang/prometheus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/registry/generic" - "k8s.io/apiserver/pkg/registry/rest" - genericapiserver "k8s.io/apiserver/pkg/server" - openapi "k8s.io/kube-openapi/pkg/common" - "k8s.io/kube-openapi/pkg/spec3" - "k8s.io/utils/strings/slices" ) var _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) @@ -204,7 +203,7 @@ func resourceFromPluginID(pluginID string) (utils.ResourceInfo, error) { return datasource.GenericConnectionResourceInfo.WithGroupAndShortName(group, pluginID+"-connection"), nil } -func (b *DataSourceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ *runtime.Scheme, _ generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *DataSourceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ builder.APIGroupOptions) error { storage := map[string]rest.Storage{} conn := b.connectionResourceInfo diff --git a/pkg/registry/apis/featuretoggle/register.go b/pkg/registry/apis/featuretoggle/register.go index 52cbb6d7a65..524bf62c0a8 100644 --- a/pkg/registry/apis/featuretoggle/register.go +++ b/pkg/registry/apis/featuretoggle/register.go @@ -5,7 +5,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" @@ -15,7 +14,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -81,7 +79,7 @@ func (b *FeatureFlagAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(gv) } -func (b *FeatureFlagAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ *runtime.Scheme, _ generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *FeatureFlagAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ builder.APIGroupOptions) error { featureStore := NewFeaturesStorage() toggleStore := NewTogglesStorage(b.features) diff --git a/pkg/registry/apis/folders/register.go b/pkg/registry/apis/folders/register.go index a891d5f249d..10239db549e 100644 --- a/pkg/registry/apis/folders/register.go +++ b/pkg/registry/apis/folders/register.go @@ -8,7 +8,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" @@ -18,7 +17,6 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/utils" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" @@ -96,7 +94,11 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(b.gv) } -func (b *FolderAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, dualWriteBuilder grafanarest.DualWriteBuilder) error { +func (b *FolderAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { + scheme := opts.Scheme + optsGetter := opts.OptsGetter + dualWriteBuilder := opts.DualWriteBuilder + legacyStore := &legacyStorage{ service: b.folderSvc, namespacer: b.namespacer, diff --git a/pkg/registry/apis/iam/register.go b/pkg/registry/apis/iam/register.go index 1ef15561291..859618b58fa 100644 --- a/pkg/registry/apis/iam/register.go +++ b/pkg/registry/apis/iam/register.go @@ -7,7 +7,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" @@ -15,7 +14,6 @@ import ( "github.com/grafana/authlib/claims" "github.com/grafana/grafana/pkg/apimachinery/identity" iamv0 "github.com/grafana/grafana/pkg/apis/iam/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/registry/apis/iam/legacy" "github.com/grafana/grafana/pkg/registry/apis/iam/serviceaccount" @@ -93,7 +91,7 @@ func (b *IdentityAccessManagementAPIBuilder) InstallSchema(scheme *runtime.Schem return scheme.SetVersionPriority(iamv0.SchemeGroupVersion) } -func (b *IdentityAccessManagementAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ *runtime.Scheme, _ generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *IdentityAccessManagementAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ builder.APIGroupOptions) error { storage := map[string]rest.Storage{} teamResource := iamv0.TeamResourceInfo diff --git a/pkg/registry/apis/peakq/register.go b/pkg/registry/apis/peakq/register.go index bb2d38c356d..54306e85daa 100644 --- a/pkg/registry/apis/peakq/register.go +++ b/pkg/registry/apis/peakq/register.go @@ -6,7 +6,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" @@ -15,7 +14,6 @@ import ( peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1" grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -65,11 +63,11 @@ func (b *PeakQAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(gv) } -func (b *PeakQAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *PeakQAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { resourceInfo := peakq.QueryTemplateResourceInfo storage := map[string]rest.Storage{} - peakqStorage, err := grafanaregistry.NewRegistryStore(scheme, resourceInfo, optsGetter) + peakqStorage, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter) if err != nil { return err } diff --git a/pkg/registry/apis/playlist/register.go b/pkg/registry/apis/playlist/register.go index dc1e3e5ea4d..d31c5218672 100644 --- a/pkg/registry/apis/playlist/register.go +++ b/pkg/registry/apis/playlist/register.go @@ -8,20 +8,19 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" + "github.com/prometheus/client_golang/prometheus" + playlist "github.com/grafana/grafana/apps/playlist/apis/playlist/v0alpha1" "github.com/grafana/grafana/pkg/apimachinery/utils" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/setting" - "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*PlaylistAPIBuilder)(nil) @@ -78,7 +77,11 @@ func (b *PlaylistAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(b.gv) } -func (b *PlaylistAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, dualWriteBuilder grafanarest.DualWriteBuilder) error { +func (b *PlaylistAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { + scheme := opts.Scheme + optsGetter := opts.OptsGetter + dualWriteBuilder := opts.DualWriteBuilder + storage := map[string]rest.Storage{} gvr := schema.GroupVersionResource{ diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go index fdd8388b961..a08af9909cc 100644 --- a/pkg/registry/apis/query/register.go +++ b/pkg/registry/apis/query/register.go @@ -8,14 +8,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/expr" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" @@ -138,7 +136,7 @@ func (b *QueryAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(query.SchemeGroupVersion) } -func (b *QueryAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ *runtime.Scheme, _ generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *QueryAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ builder.APIGroupOptions) error { gv := query.SchemeGroupVersion storage := map[string]rest.Storage{} diff --git a/pkg/registry/apis/scope/register.go b/pkg/registry/apis/scope/register.go index b41c1bea43b..8e4965666d3 100644 --- a/pkg/registry/apis/scope/register.go +++ b/pkg/registry/apis/scope/register.go @@ -6,7 +6,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" @@ -16,7 +15,6 @@ import ( "github.com/prometheus/client_golang/prometheus" scope "github.com/grafana/grafana/pkg/apis/scope/v0alpha1" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -111,7 +109,10 @@ func (b *ScopeAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(scope.SchemeGroupVersion) } -func (b *ScopeAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *ScopeAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { + scheme := opts.Scheme + optsGetter := opts.OptsGetter + scopeResourceInfo := scope.ScopeResourceInfo scopeDashboardResourceInfo := scope.ScopeDashboardBindingResourceInfo scopeNodeResourceInfo := scope.ScopeNodeResourceInfo diff --git a/pkg/registry/apis/search/register.go b/pkg/registry/apis/search/register.go index 683f9dfbcf5..c5162e5d963 100644 --- a/pkg/registry/apis/search/register.go +++ b/pkg/registry/apis/search/register.go @@ -6,18 +6,17 @@ import ( "net/url" "strconv" - "github.com/grafana/grafana/pkg/api/response" - request2 "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" - "github.com/grafana/grafana/pkg/setting" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" + "github.com/grafana/grafana/pkg/api/response" + request2 "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/storage/unified/resource" @@ -144,7 +143,7 @@ func (b *SearchAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAP return oas, nil } -func (b *SearchAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, dualWriteBuilder grafanarest.DualWriteBuilder) error { +func (b *SearchAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, _ builder.APIGroupOptions) error { apiGroupInfo.PrioritizedVersions = []schema.GroupVersion{b.GetGroupVersion()} return nil } diff --git a/pkg/registry/apis/service/register.go b/pkg/registry/apis/service/register.go index 39138660134..52c26bf3ce0 100644 --- a/pkg/registry/apis/service/register.go +++ b/pkg/registry/apis/service/register.go @@ -6,14 +6,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" service "github.com/grafana/grafana/pkg/apis/service/v0alpha1" grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic" - grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -70,7 +68,10 @@ func (b *ServiceAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { return scheme.SetVersionPriority(gv) } -func (b *ServiceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriteBuilder) error { +func (b *ServiceAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts builder.APIGroupOptions) error { + scheme := opts.Scheme + optsGetter := opts.OptsGetter + resourceInfo := service.ExternalNameResourceInfo storage := map[string]rest.Storage{} diff --git a/pkg/services/apiserver/aggregator/aggregator.go b/pkg/services/apiserver/aggregator/aggregator.go index c5eb66498c7..83080c780fc 100644 --- a/pkg/services/apiserver/aggregator/aggregator.go +++ b/pkg/services/apiserver/aggregator/aggregator.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" + "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog/v2" v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" @@ -45,15 +46,17 @@ import ( "k8s.io/kube-aggregator/pkg/controllers/autoregister" servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1" - "github.com/grafana/grafana/pkg/registry/apis/service" - servicev0alpha1applyconfiguration "github.com/grafana/grafana/pkg/generated/applyconfiguration/service/v0alpha1" serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned" informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions" + "github.com/grafana/grafana/pkg/registry/apis/service" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/apiserver/options" ) +// making sure we only register metrics once into legacy registry +var registerIntoLegacyRegistryOnce sync.Once + func readCABundlePEM(path string, devMode bool) ([]byte, error) { if devMode { return nil, nil @@ -260,6 +263,15 @@ func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.D } } + metrics := newAvailabilityMetrics() + + // create shared (remote and local) availability metrics + // TODO: decouple from legacyregistry + registerIntoLegacyRegistryOnce.Do(func() { err = metrics.Register(legacyregistry.Register, legacyregistry.CustomRegister) }) + if err != nil { + return nil, err + } + availableController, err := NewAvailableConditionController( aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), externalNamesInformer, @@ -267,6 +279,7 @@ func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.D nil, proxyCurrentCertKeyContentFunc, completedConfig.ExtraConfig.ServiceResolver, + metrics, ) if err != nil { return nil, err @@ -288,9 +301,12 @@ func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.D for _, b := range config.Builders { err := b.UpdateAPIGroupInfo( &serviceAPIGroupInfo, - aggregatorscheme.Scheme, - aggregatorConfig.GenericConfig.RESTOptionsGetter, - nil, // no dual writer + builder.APIGroupOptions{ + Scheme: aggregatorscheme.Scheme, + OptsGetter: aggregatorConfig.GenericConfig.RESTOptionsGetter, + DualWriteBuilder: nil, // no dual writer + MetricsRegister: reg, + }, ) if err != nil { return nil, err diff --git a/pkg/services/apiserver/aggregator/availableController.go b/pkg/services/apiserver/aggregator/availableController.go index 882eb353712..f881899de89 100644 --- a/pkg/services/apiserver/aggregator/availableController.go +++ b/pkg/services/apiserver/aggregator/availableController.go @@ -17,7 +17,6 @@ import ( "github.com/grafana/grafana/pkg/apis/service/v0alpha1" informersservicev0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions/service/v0alpha1" listersservicev0alpha1 "github.com/grafana/grafana/pkg/generated/listers/service/v0alpha1" - "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -69,6 +68,7 @@ type AvailableConditionController struct { cache map[string]map[string][]string // this lock protects operations on the above cache cacheLock sync.RWMutex + metrics *Metrics } // NewAvailableConditionController returns a new AvailableConditionController. @@ -79,6 +79,7 @@ func NewAvailableConditionController( proxyTransportDial *transport.DialHolder, proxyCurrentCertKeyContent certKeyFunc, serviceResolver ServiceResolver, + metrics *Metrics, ) (*AvailableConditionController, error) { c := &AvailableConditionController{ apiServiceClient: apiServiceClient, @@ -94,6 +95,7 @@ func NewAvailableConditionController( ), proxyTransportDial: proxyTransportDial, proxyCurrentCertKeyContent: proxyCurrentCertKeyContent, + metrics: metrics, } // resync on this one because it is low cardinality and rechecking the actual discovery @@ -124,6 +126,7 @@ func NewAvailableConditionController( func (c *AvailableConditionController) sync(key string) error { originalAPIService, err := c.apiServiceLister.Get(key) if apierrors.IsNotFound(err) { + c.metrics.ForgetAPIService(key) return nil } if err != nil { @@ -225,7 +228,7 @@ func (c *AvailableConditionController) sync(key string) error { _ = resp.Body.Close() // we should always been in the 200s or 300s if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - errCh <- fmt.Errorf("bad status from %v: %v", discoveryURL, resp.StatusCode) + errCh <- fmt.Errorf("bad status from %v: %d", discoveryURL, resp.StatusCode) return } } @@ -285,6 +288,9 @@ func (c *AvailableConditionController) sync(key string) error { // updateAPIServiceStatus only issues an update if a change is detected. We have a tight resync loop to quickly detect dead // apiservices. Doing that means we don't want to quickly issue no-op updates. func (c *AvailableConditionController) updateAPIServiceStatus(originalAPIService, newAPIService *apiregistrationv1.APIService) (*apiregistrationv1.APIService, error) { + // update this metric on every sync operation to reflect the actual state + c.metrics.SetUnavailableGauge(newAPIService) + if equality.Semantic.DeepEqual(originalAPIService.Status, newAPIService.Status) { return newAPIService, nil } @@ -310,6 +316,7 @@ func (c *AvailableConditionController) updateAPIServiceStatus(originalAPIService return nil, err } + c.metrics.SetUnavailableCounter(originalAPIService, newAPIService) return newAPIService, nil } diff --git a/pkg/services/apiserver/aggregator/metrics.go b/pkg/services/apiserver/aggregator/metrics.go new file mode 100644 index 00000000000..64798dad63c --- /dev/null +++ b/pkg/services/apiserver/aggregator/metrics.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kube-aggregator/pkg/controllers/status/metrics/metrics.go +// Provenance-includes-license: Apache-2.0 +// Provenance-includes-copyright: The Kubernetes Authors. + +package aggregator + +import ( + "sync" + + "k8s.io/component-base/metrics" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" +) + +/* + * By default, all the following metrics are defined as falling under + * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes) + * + * Promoting the stability level of the metric is a responsibility of the component owner, since it + * involves explicitly acknowledging support for the metric across multiple releases, in accordance with + * the metric stability policy. + */ +var ( + unavailableGaugeDesc = metrics.NewDesc( + "st_aggregator_unavailable_apiservice", + "Gauge of Grafana APIServices which are marked as unavailable broken down by APIService name.", + []string{"name"}, + nil, + metrics.ALPHA, + "", + ) +) + +type Metrics struct { + unavailableCounter *metrics.CounterVec + + *availabilityCollector +} + +func newAvailabilityMetrics() *Metrics { + return &Metrics{ + unavailableCounter: metrics.NewCounterVec( + &metrics.CounterOpts{ + // These metrics are registered in the main kube-aggregator package as well, prefixing with single-tenant (ST) to avoid + // "duplicate metrics collector registration attempted" in https://github.com/prometheus/client_golang + // a more descriptive prefix is already added for apiserver metrics during scraping in cloud and didn't want + // to double a word by using a word such as "grafana" here + Name: "st_aggregator_unavailable_apiservice_total", + Help: "Counter of Grafana APIServices which are marked as unavailable broken down by APIService name and reason.", + StabilityLevel: metrics.ALPHA, + }, + []string{"name", "reason"}, + ), + availabilityCollector: newAvailabilityCollector(), + } +} + +// Register registers apiservice availability metrics. +func (m *Metrics) Register( + registrationFunc func(metrics.Registerable) error, + customRegistrationFunc func(metrics.StableCollector) error, +) error { + err := registrationFunc(m.unavailableCounter) + if err != nil { + return err + } + + err = customRegistrationFunc(m.availabilityCollector) + if err != nil { + return err + } + + return nil +} + +// UnavailableCounter returns a counter to track apiservices marked as unavailable. +func (m *Metrics) UnavailableCounter(apiServiceName, reason string) metrics.CounterMetric { + return m.unavailableCounter.WithLabelValues(apiServiceName, reason) +} + +type availabilityCollector struct { + metrics.BaseStableCollector + + mtx sync.RWMutex + availabilities map[string]bool +} + +// SetUnavailableGauge set the metrics so that it reflect the current state based on availability of the given service +func (m *Metrics) SetUnavailableGauge(newAPIService *apiregistrationv1.APIService) { + if apiregistrationv1apihelper.IsAPIServiceConditionTrue(newAPIService, apiregistrationv1.Available) { + m.SetAPIServiceAvailable(newAPIService.Name) + return + } + + m.SetAPIServiceUnavailable(newAPIService.Name) +} + +// SetUnavailableCounter increases the metrics only if the given service is unavailable and its APIServiceCondition has changed +func (m *Metrics) SetUnavailableCounter(originalAPIService, newAPIService *apiregistrationv1.APIService) { + wasAvailable := apiregistrationv1apihelper.IsAPIServiceConditionTrue(originalAPIService, apiregistrationv1.Available) + isAvailable := apiregistrationv1apihelper.IsAPIServiceConditionTrue(newAPIService, apiregistrationv1.Available) + statusChanged := isAvailable != wasAvailable + + if statusChanged && !isAvailable { + reason := "UnknownReason" + if newCondition := apiregistrationv1apihelper.GetAPIServiceConditionByType(newAPIService, apiregistrationv1.Available); newCondition != nil { + reason = newCondition.Reason + } + m.UnavailableCounter(newAPIService.Name, reason).Inc() + } +} + +// Check if apiServiceStatusCollector implements necessary interface. +var _ metrics.StableCollector = &availabilityCollector{} + +func newAvailabilityCollector() *availabilityCollector { + return &availabilityCollector{ + availabilities: make(map[string]bool), + } +} + +// DescribeWithStability implements the metrics.StableCollector interface. +func (c *availabilityCollector) DescribeWithStability(ch chan<- *metrics.Desc) { + ch <- unavailableGaugeDesc +} + +// CollectWithStability implements the metrics.StableCollector interface. +func (c *availabilityCollector) CollectWithStability(ch chan<- metrics.Metric) { + c.mtx.RLock() + defer c.mtx.RUnlock() + + for apiServiceName, isAvailable := range c.availabilities { + gaugeValue := 1.0 + if isAvailable { + gaugeValue = 0.0 + } + ch <- metrics.NewLazyConstMetric( + unavailableGaugeDesc, + metrics.GaugeValue, + gaugeValue, + apiServiceName, + ) + } +} + +// SetAPIServiceAvailable sets the given apiservice availability gauge to available. +func (c *availabilityCollector) SetAPIServiceAvailable(apiServiceKey string) { + c.setAPIServiceAvailability(apiServiceKey, true) +} + +// SetAPIServiceUnavailable sets the given apiservice availability gauge to unavailable. +func (c *availabilityCollector) SetAPIServiceUnavailable(apiServiceKey string) { + c.setAPIServiceAvailability(apiServiceKey, false) +} + +func (c *availabilityCollector) setAPIServiceAvailability(apiServiceKey string, availability bool) { + c.mtx.Lock() + defer c.mtx.Unlock() + + c.availabilities[apiServiceKey] = availability +} + +// ForgetAPIService removes the availability gauge of the given apiservice. +func (c *availabilityCollector) ForgetAPIService(apiServiceKey string) { + c.mtx.Lock() + defer c.mtx.Unlock() + + delete(c.availabilities, apiServiceKey) +} diff --git a/pkg/services/apiserver/builder/common.go b/pkg/services/apiserver/builder/common.go index 746ee55e2c1..5e2f9b14ba9 100644 --- a/pkg/services/apiserver/builder/common.go +++ b/pkg/services/apiserver/builder/common.go @@ -11,6 +11,8 @@ import ( "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" + "github.com/prometheus/client_golang/prometheus" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" ) @@ -30,12 +32,7 @@ type APIGroupBuilder interface { // // The caller should share the apiGroupInfo passed into this function across builder versions of the same group. // UpdateAPIGroupInfo builds the group+version behavior updating the passed in apiGroupInfo in place - UpdateAPIGroupInfo( - apiGroupInfo *genericapiserver.APIGroupInfo, - scheme *runtime.Scheme, - optsGetter generic.RESTOptionsGetter, - dualWriteBuilder grafanarest.DualWriteBuilder, - ) error + UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupInfo, opts APIGroupOptions) error // Get OpenAPI definitions GetOpenAPIDefinitions() common.GetOpenAPIDefinitions @@ -49,6 +46,13 @@ type APIGroupBuilder interface { GetAuthorizer() authorizer.Authorizer } +type APIGroupOptions struct { + Scheme *runtime.Scheme + OptsGetter generic.RESTOptionsGetter + DualWriteBuilder grafanarest.DualWriteBuilder + MetricsRegister prometheus.Registerer +} + // Builders that implement OpenAPIPostProcessor are given a chance to modify the schema directly type OpenAPIPostProcessor interface { PostProcessOpenAPI(*spec3.OpenAPI) (*spec3.OpenAPI, error) diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index fd2a0e2fab9..ae5b214ecbf 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -235,7 +235,12 @@ func InstallAPIs( for group, buildersForGroup := range buildersGroupMap { g := genericapiserver.NewDefaultAPIGroupInfo(group, scheme, metav1.ParameterCodec, codecs) for _, b := range buildersForGroup { - if err := b.UpdateAPIGroupInfo(&g, scheme, optsGetter, dualWrite); err != nil { + if err := b.UpdateAPIGroupInfo(&g, APIGroupOptions{ + Scheme: scheme, + OptsGetter: optsGetter, + DualWriteBuilder: dualWrite, + MetricsRegister: reg, + }); err != nil { return err } if len(g.PrioritizedVersions) < 1 { diff --git a/pkg/services/authz/client.go b/pkg/services/authz/client.go index 4b62f3bb19f..098c1166cb1 100644 --- a/pkg/services/authz/client.go +++ b/pkg/services/authz/client.go @@ -24,7 +24,7 @@ import ( const authzServiceAudience = "authzService" type Client interface { - authzlib.MultiTenantClient + authzlib.Client } // ProvideAuthZClient provides an AuthZ client and creates the AuthZ service. @@ -41,7 +41,7 @@ func ProvideAuthZClient( return nil, err } - var client authzlib.MultiTenantClient + var client authzlib.Client // Register the server server, err := newLegacyServer(acSvc, features, grpcServer, tracer, authCfg) @@ -89,7 +89,7 @@ func ProvideStandaloneAuthZClient( return newGrpcLegacyClient(authCfg) } -func newInProcLegacyClient(server *legacyServer) (authzlib.MultiTenantClient, error) { +func newInProcLegacyClient(server *legacyServer) (authzlib.Client, error) { noAuth := func(ctx context.Context) (context.Context, error) { return ctx, nil } @@ -105,15 +105,13 @@ func newInProcLegacyClient(server *legacyServer) (authzlib.MultiTenantClient, er ) return authzlib.NewLegacyClient( - &authzlib.MultiTenantClientConfig{}, + &authzlib.ClientConfig{}, authzlib.WithGrpcConnectionLCOption(channel), - // nolint:staticcheck - authzlib.WithNamespaceFormatterLCOption(claims.OrgNamespaceFormatter), authzlib.WithDisableAccessTokenLCOption(), ) } -func newGrpcLegacyClient(authCfg *Cfg) (authzlib.MultiTenantClient, error) { +func newGrpcLegacyClient(authCfg *Cfg) (authzlib.Client, error) { // This client interceptor is a noop, as we don't send an access token clientConfig := authnlib.GrpcClientConfig{} clientInterceptor, err := authnlib.NewGrpcClientInterceptor(&clientConfig, authnlib.WithDisableAccessTokenOption()) @@ -121,13 +119,11 @@ func newGrpcLegacyClient(authCfg *Cfg) (authzlib.MultiTenantClient, error) { return nil, err } - cfg := authzlib.MultiTenantClientConfig{RemoteAddress: authCfg.remoteAddress} + cfg := authzlib.ClientConfig{RemoteAddress: authCfg.remoteAddress} client, err := authzlib.NewLegacyClient(&cfg, authzlib.WithGrpcDialOptionsLCOption( getDialOpts(clientInterceptor, authCfg.allowInsecure)..., ), - // nolint:staticcheck - authzlib.WithNamespaceFormatterLCOption(claims.OrgNamespaceFormatter), // TODO: remove this once access tokens are supported on-prem authzlib.WithDisableAccessTokenLCOption(), ) @@ -138,7 +134,7 @@ func newGrpcLegacyClient(authCfg *Cfg) (authzlib.MultiTenantClient, error) { return client, nil } -func newCloudLegacyClient(authCfg *Cfg) (authzlib.MultiTenantClient, error) { +func newCloudLegacyClient(authCfg *Cfg) (authzlib.Client, error) { grpcClientConfig := authnlib.GrpcClientConfig{ TokenClientConfig: &authnlib.TokenExchangeConfig{ Token: authCfg.token, @@ -155,7 +151,7 @@ func newCloudLegacyClient(authCfg *Cfg) (authzlib.MultiTenantClient, error) { return nil, err } - clientCfg := authzlib.MultiTenantClientConfig{RemoteAddress: authCfg.remoteAddress} + clientCfg := authzlib.ClientConfig{RemoteAddress: authCfg.remoteAddress} client, err := authzlib.NewLegacyClient(&clientCfg, authzlib.WithGrpcDialOptionsLCOption( getDialOpts(clientInterceptor, authCfg.allowInsecure)..., diff --git a/pkg/services/authz/server.go b/pkg/services/authz/server.go index 8ca9d25d47e..7df802aa758 100644 --- a/pkg/services/authz/server.go +++ b/pkg/services/authz/server.go @@ -2,8 +2,10 @@ package authz import ( "context" + "fmt" authzv1 "github.com/grafana/authlib/authz/proto/v1" + "github.com/grafana/authlib/claims" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" @@ -79,14 +81,19 @@ func (s *legacyServer) Read(ctx context.Context, req *authzv1.ReadRequest) (*aut action := req.GetAction() subject := req.GetSubject() - stackID := req.GetStackId() // TODO can we consider the stackID as the orgID? + namespace := req.GetNamespace() // TODO can we consider the stackID as the orgID? + + info, err := claims.ParseNamespace(namespace) + if err != nil || info.OrgID == 0 { + return nil, fmt.Errorf("invalid namespace: %s", namespace) + } ctxLogger := s.logger.FromContext(ctx) - ctxLogger.Debug("Read", "action", action, "subject", subject, "stackID", stackID) + ctxLogger.Debug("Read", "action", action, "subject", subject, "namespace", namespace) permissions, err := s.acSvc.SearchUserPermissions( ctx, - stackID, + info.OrgID, accesscontrol.SearchOptions{Action: action, TypedID: subject}, ) if err != nil { @@ -96,7 +103,7 @@ func (s *legacyServer) Read(ctx context.Context, req *authzv1.ReadRequest) (*aut data := make([]*authzv1.ReadResponse_Data, 0, len(permissions)) for _, perm := range permissions { - data = append(data, &authzv1.ReadResponse_Data{Object: perm.Scope}) + data = append(data, &authzv1.ReadResponse_Data{Scope: perm.Scope}) } return &authzv1.ReadResponse{ Data: data, diff --git a/pkg/services/authz/zanzana.go b/pkg/services/authz/zanzana.go index 1c83006e7c4..c47e680f709 100644 --- a/pkg/services/authz/zanzana.go +++ b/pkg/services/authz/zanzana.go @@ -49,7 +49,7 @@ func ProvideZanzana(cfg *setting.Cfg, db db.DB, features featuremgmt.FeatureTogg return nil, fmt.Errorf("failed to start zanzana: %w", err) } - srv, err := zanzana.NewServer(store, logger) + srv, err := zanzana.NewServer(cfg, store, logger) if err != nil { return nil, fmt.Errorf("failed to start zanzana: %w", err) } @@ -104,7 +104,7 @@ func (z *Zanzana) start(ctx context.Context) error { return fmt.Errorf("failed to initilize zanana store: %w", err) } - srv, err := zanzana.NewServer(store, z.logger) + srv, err := zanzana.NewServer(z.cfg, store, z.logger) if err != nil { return fmt.Errorf("failed to start zanzana: %w", err) } diff --git a/pkg/services/authz/zanzana/client/client_test.go b/pkg/services/authz/zanzana/client/client_test.go index 050a0cafcce..35bdde46088 100644 --- a/pkg/services/authz/zanzana/client/client_test.go +++ b/pkg/services/authz/zanzana/client/client_test.go @@ -107,7 +107,7 @@ func zanzanaServerIntegrationTest(tb testing.TB) *inprocgrpc.Channel { store, err := zstore.NewEmbeddedStore(cfg, db, logger) require.NoError(tb, err) - srv, err := zserver.New(store, logger) + srv, err := zserver.New(&cfg.Zanzana, store, logger) require.NoError(tb, err) channel := &inprocgrpc.Channel{} diff --git a/pkg/services/authz/zanzana/server.go b/pkg/services/authz/zanzana/server.go index 3d6b0dc4a6a..33730ae9341 100644 --- a/pkg/services/authz/zanzana/server.go +++ b/pkg/services/authz/zanzana/server.go @@ -11,8 +11,8 @@ import ( zserver "github.com/grafana/grafana/pkg/services/authz/zanzana/server" ) -func NewServer(store storage.OpenFGADatastore, logger log.Logger) (*server.Server, error) { - return zserver.New(store, logger) +func NewServer(cfg *setting.Cfg, store storage.OpenFGADatastore, logger log.Logger) (*server.Server, error) { + return zserver.New(&cfg.Zanzana, store, logger) } func StartOpenFGAHttpSever(cfg *setting.Cfg, srv grpcserver.Provider, logger log.Logger) error { diff --git a/pkg/services/authz/zanzana/server/server.go b/pkg/services/authz/zanzana/server/server.go index 49baf954fde..dc1f935c810 100644 --- a/pkg/services/authz/zanzana/server/server.go +++ b/pkg/services/authz/zanzana/server/server.go @@ -27,11 +27,14 @@ import ( zlogger "github.com/grafana/grafana/pkg/services/authz/zanzana/logger" ) -func New(store storage.OpenFGADatastore, logger log.Logger) (*server.Server, error) { - // FIXME(kalleep): add support for more options, tracing etc +func New(cfg *setting.ZanzanaSettings, store storage.OpenFGADatastore, logger log.Logger) (*server.Server, error) { opts := []server.OpenFGAServiceV1Option{ server.WithDatastore(store), server.WithLogger(zlogger.New(logger)), + server.WithCheckQueryCacheEnabled(cfg.CheckQueryCache), + server.WithCheckQueryCacheTTL(cfg.CheckQueryCacheTTL), + server.WithListObjectsMaxResults(cfg.ListObjectsMaxResults), + server.WithListObjectsDeadline(cfg.ListObjectsDeadline), } // FIXME(kalleep): Interceptors diff --git a/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go b/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go index d087684946b..15438259240 100644 --- a/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go +++ b/pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go @@ -34,6 +34,7 @@ import ( libraryelements "github.com/grafana/grafana/pkg/services/libraryelements/model" "github.com/grafana/grafana/pkg/services/ngalert" "github.com/grafana/grafana/pkg/services/ngalert/metrics" + "github.com/grafana/grafana/pkg/services/ngalert/models" ngalertstore "github.com/grafana/grafana/pkg/services/ngalert/store" ngalertfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" @@ -773,12 +774,16 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi }, } - featureToggles := featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations, featuremgmt.FlagDashboardRestore) + featureToggles := featuremgmt.WithFeatures( + featuremgmt.FlagOnPremToCloudMigrations, + featuremgmt.FlagOnPremToCloudMigrationsAlerts, + featuremgmt.FlagDashboardRestore, // needed for skipping creating soft-deleted dashboards in the snapshot. + ) kvStore := kvstore.ProvideService(sqlStore) bus := bus.ProvideBus(tracer) - fakeAccessControl := actest.FakeAccessControl{} + fakeAccessControl := actest.FakeAccessControl{ExpectedEvaluate: true} fakeAccessControlService := actest.FakeService{} alertMetrics := metrics.NewNGAlert(prometheus.NewRegistry()) @@ -793,12 +798,34 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi ) require.NoError(t, err) + var validConfig = `{ + "alertmanager_config": { + "route": { + "receiver": "grafana-default-email" + }, + "receivers": [{ + "name": "grafana-default-email", + "grafana_managed_receiver_configs": [{ + "uid": "", + "name": "email receiver", + "type": "email", + "settings": { + "addresses": "" + } + }] + }] + } + }` + require.NoError(t, ng.Api.AlertingStore.SaveAlertmanagerConfiguration(context.Background(), &models.SaveAlertmanagerConfigurationCmd{ + AlertmanagerConfiguration: validConfig, + OrgID: 1, + LastApplied: time.Now().Unix(), + })) + s, err := ProvideService( cfg, httpclient.NewProvider(), - featuremgmt.WithFeatures( - featuremgmt.FlagOnPremToCloudMigrations, - featuremgmt.FlagDashboardRestore), + featureToggles, sqlStore, dsService, secretskv.NewFakeSQLSecretsKVStore(t, sqlStore), diff --git a/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go b/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go index 5c9df830110..fd977a85be7 100644 --- a/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go +++ b/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go @@ -32,6 +32,9 @@ var currentMigrationTypes = []cloudmigration.MigrateDataType{ cloudmigration.FolderDataType, cloudmigration.LibraryElementDataType, cloudmigration.DashboardDataType, + cloudmigration.MuteTimingType, + cloudmigration.NotificationTemplateType, + cloudmigration.ContactPointType, } func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.SignedInUser) (*cloudmigration.MigrateDataRequest, error) { @@ -58,9 +61,31 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S return nil, err } + // Alerts: Mute Timings + muteTimings, err := s.getAlertMuteTimings(ctx, signedInUser) + if err != nil { + s.log.Error("Failed to get alert mute timings", "err", err) + return nil, err + } + + // Alerts: Notification Templates + notificationTemplates, err := s.getNotificationTemplates(ctx, signedInUser) + if err != nil { + s.log.Error("Failed to get alert notification templates", "err", err) + return nil, err + } + + // Alerts: Contact Points + contactPoints, err := s.getContactPoints(ctx, signedInUser) + if err != nil { + s.log.Error("Failed to get alert contact points", "err", err) + return nil, err + } + migrationDataSlice := make( []cloudmigration.MigrateDataRequestItem, 0, - len(dataSources)+len(dashs)+len(folders)+len(libraryElements), + len(dataSources)+len(dashs)+len(folders)+len(libraryElements)+ + len(muteTimings)+len(notificationTemplates)+len(contactPoints), ) for _, ds := range dataSources { @@ -107,6 +132,33 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S }) } + for _, muteTiming := range muteTimings { + migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{ + Type: cloudmigration.MuteTimingType, + RefID: muteTiming.Name, + Name: muteTiming.Name, + Data: muteTiming, + }) + } + + for _, notificationTemplate := range notificationTemplates { + migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{ + Type: cloudmigration.NotificationTemplateType, + RefID: notificationTemplate.Name, + Name: notificationTemplate.Name, + Data: notificationTemplate, + }) + } + + for _, contactPoint := range contactPoints { + migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{ + Type: cloudmigration.ContactPointType, + RefID: contactPoint.UID, + Name: contactPoint.Name, + Data: contactPoint, + }) + } + // Obtain the names of parent elements for Dashboard and Folders data types parentNamesByType, err := s.getParentNames(ctx, signedInUser, dashs, folders, libraryElements) if err != nil { diff --git a/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt_alerts.go b/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt_alerts.go new file mode 100644 index 00000000000..afc2ca5ec80 --- /dev/null +++ b/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt_alerts.go @@ -0,0 +1,108 @@ +package cloudmigrationimpl + +import ( + "context" + "fmt" + + "github.com/prometheus/alertmanager/config" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/ngalert/provisioning" + "github.com/grafana/grafana/pkg/services/user" +) + +type muteTimeInterval struct { + // There is a lot of custom (de)serialization logic from Alertmanager, + // and this is the same type used by the underlying API, hence we can use the type as-is. + config.MuteTimeInterval `json:",inline"` +} + +func (s *Service) getAlertMuteTimings(ctx context.Context, signedInUser *user.SignedInUser) ([]muteTimeInterval, error) { + if !s.features.IsEnabledGlobally(featuremgmt.FlagOnPremToCloudMigrationsAlerts) { + return nil, nil + } + + muteTimings, err := s.ngAlert.Api.MuteTimings.GetMuteTimings(ctx, signedInUser.OrgID) + if err != nil { + return nil, fmt.Errorf("fetching ngalert mute timings: %w", err) + } + + muteTimeIntervals := make([]muteTimeInterval, 0, len(muteTimings)) + + for _, muteTiming := range muteTimings { + muteTimeIntervals = append(muteTimeIntervals, muteTimeInterval{ + MuteTimeInterval: config.MuteTimeInterval{ + Name: muteTiming.Name, + TimeIntervals: muteTiming.TimeIntervals, + }, + }) + } + + return muteTimeIntervals, nil +} + +type notificationTemplate struct { + Name string `json:"name"` + Template string `json:"template"` +} + +func (s *Service) getNotificationTemplates(ctx context.Context, signedInUser *user.SignedInUser) ([]notificationTemplate, error) { + if !s.features.IsEnabledGlobally(featuremgmt.FlagOnPremToCloudMigrationsAlerts) { + return nil, nil + } + + templates, err := s.ngAlert.Api.Templates.GetTemplates(ctx, signedInUser.OrgID) + if err != nil { + return nil, fmt.Errorf("fetching ngalert notification templates: %w", err) + } + + notificationTemplates := make([]notificationTemplate, 0, len(templates)) + + for _, template := range templates { + notificationTemplates = append(notificationTemplates, notificationTemplate{ + Name: template.Name, + Template: template.Template, + }) + } + + return notificationTemplates, nil +} + +type contactPoint struct { + Settings *simplejson.Json `json:"settings"` + UID string `json:"uid"` + Name string `json:"name"` + Type string `json:"type"` + DisableResolveMessage bool `json:"disableResolveMessage"` +} + +func (s *Service) getContactPoints(ctx context.Context, signedInUser *user.SignedInUser) ([]contactPoint, error) { + if !s.features.IsEnabledGlobally(featuremgmt.FlagOnPremToCloudMigrationsAlerts) { + return nil, nil + } + + query := provisioning.ContactPointQuery{ + OrgID: signedInUser.GetOrgID(), + Decrypt: true, // needed to recreate the settings in the target instance. + } + + embeddedContactPoints, err := s.ngAlert.Api.ContactPointService.GetContactPoints(ctx, query, signedInUser) + if err != nil { + return nil, fmt.Errorf("fetching ngalert contact points: %w", err) + } + + contactPoints := make([]contactPoint, 0, len(embeddedContactPoints)) + + for _, embeddedContactPoint := range embeddedContactPoints { + contactPoints = append(contactPoints, contactPoint{ + UID: embeddedContactPoint.UID, + Name: embeddedContactPoint.Name, + Type: embeddedContactPoint.Type, + Settings: embeddedContactPoint.Settings, + DisableResolveMessage: embeddedContactPoint.DisableResolveMessage, + }) + } + + return contactPoints, nil +} diff --git a/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt_alerts_test.go b/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt_alerts_test.go new file mode 100644 index 00000000000..058f7e5b257 --- /dev/null +++ b/pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt_alerts_test.go @@ -0,0 +1,207 @@ +package cloudmigrationimpl + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/featuremgmt" + ac "github.com/grafana/grafana/pkg/services/ngalert/accesscontrol" + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/services/user" +) + +func TestGetAlertMuteTimings(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + t.Run("when the feature flag `onPremToCloudMigrationsAlerts` is not enabled it returns nil", func(t *testing.T) { + s := setUpServiceTest(t, false).(*Service) + s.features = featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations) + + muteTimeIntervals, err := s.getAlertMuteTimings(ctx, nil) + require.NoError(t, err) + require.Nil(t, muteTimeIntervals) + }) + + t.Run("when the feature flag `onPremToCloudMigrationsAlerts` is enabled it returns the mute timings", func(t *testing.T) { + s := setUpServiceTest(t, false).(*Service) + s.features = featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations, featuremgmt.FlagOnPremToCloudMigrationsAlerts) + + var orgID int64 = 1 + user := &user.SignedInUser{OrgID: orgID} + + createdMuteTiming := createMuteTiming(t, ctx, s, orgID) + + muteTimeIntervals, err := s.getAlertMuteTimings(ctx, user) + require.NoError(t, err) + require.NotNil(t, muteTimeIntervals) + require.Len(t, muteTimeIntervals, 1) + require.Equal(t, createdMuteTiming.Name, muteTimeIntervals[0].Name) + }) +} + +func TestGetNotificationTemplates(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + t.Run("when the feature flag `onPremToCloudMigrationsAlerts` is not enabled it returns nil", func(t *testing.T) { + s := setUpServiceTest(t, false).(*Service) + s.features = featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations) + + notificationTemplates, err := s.getNotificationTemplates(ctx, nil) + require.NoError(t, err) + require.Nil(t, notificationTemplates) + }) + + t.Run("when the feature flag `onPremToCloudMigrationsAlerts` is enabled it returns the notification templates", func(t *testing.T) { + s := setUpServiceTest(t, false).(*Service) + s.features = featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations, featuremgmt.FlagOnPremToCloudMigrationsAlerts) + + var orgID int64 = 1 + user := &user.SignedInUser{OrgID: orgID} + + createdTemplate := createNotificationTemplate(t, ctx, s, orgID) + + notificationTemplates, err := s.getNotificationTemplates(ctx, user) + require.NoError(t, err) + require.NotNil(t, notificationTemplates) + require.Len(t, notificationTemplates, 1) + require.Equal(t, createdTemplate.Name, notificationTemplates[0].Name) + }) +} + +func TestGetContactPoints(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + t.Run("when the feature flag `onPremToCloudMigrationsAlerts` is not enabled it returns nil", func(t *testing.T) { + s := setUpServiceTest(t, false).(*Service) + s.features = featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations) + + contactPoints, err := s.getContactPoints(ctx, nil) + require.NoError(t, err) + require.Nil(t, contactPoints) + }) + + t.Run("when the feature flag `onPremToCloudMigrationsAlerts` is enabled it returns the contact points", func(t *testing.T) { + s := setUpServiceTest(t, false).(*Service) + s.features = featuremgmt.WithFeatures(featuremgmt.FlagOnPremToCloudMigrations, featuremgmt.FlagOnPremToCloudMigrationsAlerts) + + user := &user.SignedInUser{ + OrgID: 1, + Permissions: map[int64]map[string][]string{ + 1: { + accesscontrol.ActionAlertingNotificationsRead: nil, + accesscontrol.ActionAlertingReceiversReadSecrets: {ac.ScopeReceiversAll}, + }, + }, + } + + defaultEmailContactPointCount := 1 + + createdContactPoints := createContactPoints(t, ctx, s, user) + + contactPoints, err := s.getContactPoints(ctx, user) + require.NoError(t, err) + require.NotNil(t, contactPoints) + require.Len(t, contactPoints, len(createdContactPoints)+defaultEmailContactPointCount) + }) +} + +func createMuteTiming(t *testing.T, ctx context.Context, service *Service, orgID int64) definitions.MuteTimeInterval { + t.Helper() + + muteTiming := `{ + "name": "My Unique MuteTiming 1", + "time_intervals": [ + { + "times": [{"start_time": "12:12","end_time": "23:23"}], + "weekdays": ["monday","wednesday","friday","sunday"], + "days_of_month": ["10:20","25:-1"], + "months": ["1:6","10:12"], + "years": ["2022:2054"], + "location": "Africa/Douala" + } + ] + }` + + var mt definitions.MuteTimeInterval + require.NoError(t, json.Unmarshal([]byte(muteTiming), &mt)) + + createdTiming, err := service.ngAlert.Api.MuteTimings.CreateMuteTiming(ctx, mt, orgID) + require.NoError(t, err) + + return createdTiming +} + +func createNotificationTemplate(t *testing.T, ctx context.Context, service *Service, orgID int64) definitions.NotificationTemplate { + t.Helper() + + tmpl := definitions.NotificationTemplate{ + Name: "MyTestNotificationTemplate", + Template: "This is a test template\n{{ .ExternalURL }}", + } + + createdTemplate, err := service.ngAlert.Api.Templates.CreateTemplate(ctx, orgID, tmpl) + require.NoError(t, err) + + return createdTemplate +} + +func createContactPoints(t *testing.T, ctx context.Context, service *Service, user *user.SignedInUser) []definitions.EmbeddedContactPoint { + t.Helper() + + slackSettings, err := simplejson.NewJson([]byte(`{ + "icon_emoji":"iconemoji", + "icon_url":"iconurl", + "recipient":"recipient", + "token":"slack-secret", + "username":"user" + }`)) + require.NoError(t, err) + + telegramSettings, err := simplejson.NewJson([]byte(`{ + "bottoken":"telegram-secret", + "chatid":"chat-id", + "disable_notification":true, + "disable_web_page_preview":false, + "message_thread_id":"1234", + "parse_mode":"None", + "protect_content":true + }`)) + require.NoError(t, err) + + nameGroup := "group_1" + + slackContactPoint := definitions.EmbeddedContactPoint{ + Name: nameGroup, + Type: "slack", + Settings: slackSettings, + DisableResolveMessage: false, + Provenance: "", + } + + createdSlack, err := service.ngAlert.Api.ContactPointService.CreateContactPoint(ctx, user.GetOrgID(), user, slackContactPoint, "") + require.NoError(t, err) + + telegramContactPoint := definitions.EmbeddedContactPoint{ + Name: nameGroup, + Type: "telegram", + Settings: telegramSettings, + DisableResolveMessage: false, + Provenance: "", + } + + createdTelegram, err := service.ngAlert.Api.ContactPointService.CreateContactPoint(ctx, user.GetOrgID(), user, telegramContactPoint, "") + require.NoError(t, err) + + return []definitions.EmbeddedContactPoint{ + createdSlack, + createdTelegram, + } +} diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 9034ce2fb89..d9766ec870e 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -743,6 +743,13 @@ var ( Owner: grafanaAppPlatformSquad, FrontendOnly: true, }, + { + Name: "kubernetesDashboardsAPI", + Description: "Use the kubernetes API in the backend for dashboards", + Stage: FeatureStageExperimental, + Owner: grafanaAppPlatformSquad, + RequiresRestart: true, // changes the API routing + }, { Name: "kubernetesFolders", Description: "Use the kubernetes API in the frontend for folders, and route /api/folders requests to k8s", diff --git a/pkg/services/featuremgmt/toggles-gitlog.csv b/pkg/services/featuremgmt/toggles-gitlog.csv index a0c15aada46..7ef568102f2 100644 --- a/pkg/services/featuremgmt/toggles-gitlog.csv +++ b/pkg/services/featuremgmt/toggles-gitlog.csv @@ -46,7 +46,7 @@ export,2022-04-25T23:59:18Z,2023-03-01T17:42:53Z,e0aeb83786731769e870d79c0d8a21e savedItems,2022-04-26T10:27:01Z,2022-06-27T14:41:00Z,e420252d45cd48fdcf5cc68ad84f89e36d5c936c,Ashley Harrison tracing,2022-04-26T13:31:27Z,2023-03-21T09:35:21Z,3b4d237ade03e67a971ca8899e8e0371e1d83f2a,kay delaney cloudWatchDynamicLabels,2022-04-29T09:43:04Z,2023-05-02T08:48:17Z,7bb4f5cd9baff81b354e4bb65fd4b6141e3488f6,Shirley -datasourceQueryMultiStatus,2022-05-03T16:02:20Z,,4ecd57f49c75d72927be62aba23a15dca02f8eb6,Will Browne +datasourceQueryMultiStatus,2022-05-03T16:02:20Z,2024-07-10T09:15:10Z,4ecd57f49c75d72927be62aba23a15dca02f8eb6,Will Browne azureMonitorExperimentalUI,2022-05-04T13:54:09Z,2022-07-14T13:07:31Z,b2644de6c8a08181cbe0a9f545393bd42753477c,Adam Simpson traceToMetrics,2022-05-05T20:46:18Z,2024-02-22T13:30:41Z,c1b5ea3e54169c7a4bbd58bd8be2c4c192a093c0,Connor Lindsey prometheusStreamingJSONParser,2022-05-13T18:28:54Z,2022-11-11T16:53:12Z,87e8521591534bcfd8a7c78fbd39b226c13a99b2,Todd Treece @@ -128,7 +128,7 @@ clientTokenRotation,2023-03-23T13:39:04Z,2024-02-16T14:03:37Z,382b24742ab8b124af disableAngular,2023-03-23T15:43:45Z,,58704390269c8eeedd32c571245d5ef08f0ffb82,Ryan McKinley disableElasticsearchBackendExploreQuery,2023-03-27T13:52:27Z,2023-04-12T12:20:43Z,f3da91f53fa32a8e870124e9514a010b97359529,Ivana Huckova emptyDashboardPage,2023-03-28T09:42:23Z,2024-01-25T14:04:29Z,221c5efedc593fb42d87c849b3d0827e325fc6cd,Polina Boneva -prometheusDataplane,2023-03-29T15:26:32Z,,674144c8e84cb4aca3bfc448f08b88bb516d3b77,Kyle Brandt +prometheusDataplane,2023-03-29T15:26:32Z,2024-08-26T12:53:38Z,674144c8e84cb4aca3bfc448f08b88bb516d3b77,Kyle Brandt alertStateHistoryLokiOnly,2023-03-30T18:53:21Z,,b2abb6328677a90e3f3bc8b706d22734696d02e3,Alexander Weaver alertStateHistoryLokiPrimary,2023-03-30T18:53:21Z,,b2abb6328677a90e3f3bc8b706d22734696d02e3,Alexander Weaver alertStateHistoryLokiSecondary,2023-03-30T18:53:21Z,,b2abb6328677a90e3f3bc8b706d22734696d02e3,Alexander Weaver @@ -206,20 +206,20 @@ lokiRunQueriesInParallel,2023-09-19T09:34:01Z,,98aa7db64ab4c4d0f3699a9bcbde98944 pluginsAPIMetrics,2023-09-21T11:36:32Z,,8e8bd2760b8c05df9015900ebdf865c1629881de,Esteban Beltran externalCorePlugins,2023-09-22T08:50:13Z,,61cdfba87a36bd4b8e1bb227a14c5d88fc943aff,Andres Martinez Gotor httpSLOLevels,2023-09-22T08:52:28Z,2024-02-06T08:29:41Z,e5fbc4a4cd587e4b342afd0ab0136754cf8ebf76,Carl Bergquist -idForwarding,2023-09-25T15:21:28Z,,d15661c726b582212e1329f0ebe938664445e44a,Karl Persson +idForwarding,2023-09-25T15:21:28Z,2024-08-21T13:30:17Z,d15661c726b582212e1329f0ebe938664445e44a,Karl Persson cloudwatchNewRegionsHandler,2023-09-25T18:19:12Z,2024-01-30T12:11:52Z,ef441f02d09730449f3f7bbdcf3b31dc00ed797a,Sarah Zinger cloudWatchWildCardDimensionValues,2023-09-27T14:41:48Z,2024-03-15T13:49:53Z,06a35f55ac56ca5699031b2af9202ac7a26918fc,Isabella Siu externalServiceAccounts,2023-09-28T07:26:37Z,,4563fc48afe81ff1af54b6a5bf6d65355f961155,Gabriel MABILLE alertingModifiedExport,2023-09-28T14:07:45Z,2023-10-12T17:17:32Z,169c5262a5e7d04234119a3733b00bb787b8bf90,Sonia Aguilar enableNativeHTTPHistogram,2023-10-03T18:23:55Z,,0fc403d116b9cbd0b93e497cff0ddded55f9fe60,Carl Bergquist transformationsVariableSupport,2023-10-04T14:28:46Z,,40cdb3033697e8d8e43d0ee0342f0d16fac7d633,Oscar Kilhed -kubernetesPlaylists,2023-10-05T19:00:36Z,,664ebf771e2a1d2f94bc14d3dababb634c4ac40f,Todd Treece +kubernetesPlaylists,2023-10-05T19:00:36Z,2024-08-13T08:03:28Z,664ebf771e2a1d2f94bc14d3dababb634c4ac40f,Todd Treece grafanaAPIServerWithExperimentalAPIs,2023-10-06T18:55:22Z,,717a9dd6160e352ff7c184bfcddf2410eed9d908,Ryan McKinley panelMonitoring,2023-10-09T05:19:08Z,,ef82767dabea7d19cd8efac0c36ed590d4d767f4,Victor Marin navAdminSubsections,2023-10-10T10:50:44Z,2023-11-17T10:04:34Z,f56cc6fdc01dbf2efb802f22650cb2bef0c40179,Ashley Harrison recoveryThreshold,2023-10-10T14:51:50Z,,810fbc3327841da6d21f945e75cad9daf040e625,Yuri Tseretyan libraryPanelRBAC,2023-10-11T23:30:50Z,,a12cb8cbf3a9b33841b2f2cb1522be11de78c86a,kay delaney -awsDatasourcesNewFormStyling,2023-10-12T08:59:10Z,,2771fb940342aa152377b26b9554eb15082f90ac,Ida Štambuk +awsDatasourcesNewFormStyling,2023-10-12T08:59:10Z,2024-07-22T12:48:17Z,2771fb940342aa152377b26b9554eb15082f90ac,Ida Štambuk cachingOptimizeSerializationMemoryUsage,2023-10-12T16:56:49Z,,94ce87571ddfcede0fb7a229a65502b385d5bca3,Michael Mandrus panelTitleSearchInV1,2023-10-13T12:04:24Z,,bf2f2540da7a4e4b8d80e1fa4ae3d05868cf7b69,Arati R exploreContentOutline,2023-10-13T16:57:13Z,2024-06-24T15:45:42Z,4ec54bc2c39ba43843c693fdb2a4529b6a4703f2,Haris Rozajac @@ -245,7 +245,7 @@ pdfTables,2023-11-06T13:39:22Z,,95b48339f89c7d267bce7d894404d26ccd75d0e3,Agnès newVizTooltips,2023-11-06T16:35:59Z,2024-04-03T00:32:01Z,6b4b7127544865b78f712d907e6f1719595f4232,Adela Almasan ssoSettingsApi,2023-11-08T09:50:01Z,,5285e9503be5702680acb2b52a6bda0632f4603d,Misi logsInfiniteScrolling,2023-11-09T10:54:03Z,,174c2ab45a2af912519153c5c3e671f04396d7d7,Matias Chomicki -flameGraphItemCollapsing,2023-11-09T14:31:07Z,,494a07b522df4e3ff9512b47767745a94c15f080,Andrej Ocenas +flameGraphItemCollapsing,2023-11-09T14:31:07Z,2024-07-15T12:45:41Z,494a07b522df4e3ff9512b47767745a94c15f080,Andrej Ocenas alertingDetailsViewV2,2023-11-09T17:35:03Z,2024-03-14T14:18:01Z,323ee7c38ceb18b8e71c780d797fabac7041a673,Gilles De Mey alertingSimplifiedRouting,2023-11-10T13:14:39Z,,68e37c3925080cf64a5e7570eabb042f19ca2dbf,Sonia Aguilar dashboardScene,2023-11-13T08:51:21Z,,4bc322ca1d6ed63d7e79eecb1ed09f3043f9aedb,Torkel Ödegaard @@ -259,7 +259,7 @@ displayAnonymousStats,2023-11-29T16:58:41Z,2024-02-23T15:53:37Z,59bdff0280d52ca5 influxqlStreamingParser,2023-11-29T17:29:35Z,,5845f140758473ab5ffe789bec4077032fd22839,ismail simsek kubernetesSnapshots,2023-12-05T22:31:49Z,,439edebcd605a1b63bf3a9b0ab5c2b83341cd5cd,Ryan McKinley grafanaAPIServerEnsureKubectlAccess,2023-12-06T20:21:21Z,,c4c9bfaf2e7fa12a8e453df0f089c8b4f914a3d3,Dan Cech -unifiedStorage,2023-12-06T20:21:21Z,,c4c9bfaf2e7fa12a8e453df0f089c8b4f914a3d3,Dan Cech +unifiedStorage,2023-12-06T20:21:21Z,2024-08-21T16:28:30Z,c4c9bfaf2e7fa12a8e453df0f089c8b4f914a3d3,Dan Cech alertStateHistoryAnnotationsFromLoki,2023-12-11T19:17:01Z,2024-01-25T17:56:09Z,4c1bf86ae11696277025296b86e8514386b4bb31,William Wernert tableSharedCrosshair,2023-12-13T09:33:14Z,,5aff3389f4633d6970eb2b3629ac015e564c626c,Victor Marin lokiQueryHints,2023-12-18T20:43:16Z,,2165c9b3f000f59c9fbda80d2bfe3bc74cd6d9dc,Sven Grossmann @@ -334,7 +334,7 @@ pinNavItems,2024-06-10T11:40:03Z,,84b638fb26cecf856374bb3d09b123061b4b8a6b,Laura authZGRPCServer,2024-06-13T09:41:35Z,,afcb5a855c26e985e43861bff6fab36b1b008109,Gabriel MABILLE openSearchBackendFlowEnabled,2024-06-17T09:41:50Z,,ab2af9b8f75cd13595f4d487c1168e849768a518,Ida Štambuk ssoSettingsLDAP,2024-06-18T11:31:27Z,,d074cc7892b96a1333bd07011baff146ea71e21d,Mihai Doarna -databaseReadReplica,2024-06-18T15:07:15Z,,50244ed4a1435cbf3e3c87d4af34fd7937f7c259,Kristin Laemmert +databaseReadReplica,2024-06-18T15:07:15Z,2024-09-25T23:21:39Z,50244ed4a1435cbf3e3c87d4af34fd7937f7c259,Kristin Laemmert disableClassicHTTPHistogram,2024-06-18T19:37:44Z,,3bbc821131f1b10ace139dbb4a6880fb77686646,Dave Henderson zanzana,2024-06-19T13:59:47Z,,3fe29809bec39239c45d672d686392725773f2e1,Karl Persson failWrongDSUID,2024-06-20T10:56:39Z,,44fd13c742e606b8409e23eb62cab8bab24310f1,Andres Martinez Gotor @@ -342,5 +342,28 @@ passScopeToDashboardApi,2024-06-20T15:49:19Z,,543e71eb2862187d12e8ee7742badb06e4 alertingApiServer,2024-06-20T20:52:03Z,,b07592620279f16b0353444e7aba3c457c50d7ec,Yuri Tseretyan dashboardRestoreUI,2024-06-25T14:43:13Z,,a3879e02bb3b7e8e917ba1bb4163bb230f917f2c,Laura Fernández cloudWatchRoundUpEndTime,2024-06-27T15:10:28Z,,ba5b33227c343cb2c7dad15ff85a745869c68da9,Ida Štambuk -bodyScrolling,2024-07-01T10:28:39Z,,c0058f9c7e390d8a196f5b375382334287633ea9,Ashley Harrison +bodyScrolling,2024-07-01T10:28:39Z,2024-09-24T12:23:18Z,c0058f9c7e390d8a196f5b375382334287633ea9,Ashley Harrison cloudwatchMetricInsightsCrossAccount,2024-07-02T10:34:12Z,,36ff0fe63a7710eb496f2f048feb71e6eb6e3c56,Ida Štambuk +dataplaneAggregator,2024-08-09T08:41:07Z,,122e291134c689ff57eae4461cd5953914b5a36c,Todd Treece +adhocFilterOneOf,2024-08-12T08:56:42Z,2024-09-05T12:49:24Z,ab3e8652aa865e43a3f2c164b626c42dfffab33e,Ashley Harrison +prometheusRunQueriesInParallel,2024-08-12T12:31:39Z,,c9ddc688a2b2c4ebb49e8ecf71dc01e33704da32,Vijay Samuel +backgroundPluginInstaller,2024-08-12T14:39:31Z,2024-09-23T13:49:18Z,d342e76f636e3a5751c5e7baccd1c8910b50d863,Andres Martinez Gotor +pluginsDetailsRightPanel,2024-08-13T09:55:30Z,,8044cb50f17a021a32e7ac0b83cf8d723457a9ae,Yulia Shanyrova +lokiSendDashboardPanelNames,2024-08-22T19:30:43Z,,ec857e1de99d228668e3fb2c0bfd30230a96e180,Sven Grossmann +mysqlParseTime,2024-08-27T11:16:04Z,,c59dddf7afb227f756259a50fc1d2b1946a4a72f,Ryan McKinley +singleTopNav,2024-08-29T08:48:32Z,,8aaa155cb0215119d2455732a9e37f7c2aeff35c,Laura Fernández +exploreLogsAggregatedMetrics,2024-08-29T13:55:59Z,,15a4ff992bd925f5714bc4e73fa9766a995b5711,Sven Grossmann +exploreLogsLimitedTimeRange,2024-08-29T13:55:59Z,,15a4ff992bd925f5714bc4e73fa9766a995b5711,Sven Grossmann +exploreLogsShardSplitting,2024-08-29T13:55:59Z,,15a4ff992bd925f5714bc4e73fa9766a995b5711,Sven Grossmann +newFiltersUI,2024-08-30T12:48:13Z,,00ae49a61adff11e5fbac3341941b35185d9e8d9,Sergej-Vlasov +appPlatformAccessTokens,2024-09-05T16:18:44Z,,d5ebaa0ef92edecfb9511dc7c8080be25f8cc7e7,Claudiu Dragalina-Paraipan +appSidecar,2024-09-09T12:45:05Z,,5e2ac24890906e5070323d87730dd78a4f885963,Andrej Ocenas +vizActions,2024-09-09T14:11:55Z,,af48d3db1eb2d8681843f5997e50fea5e5ea3096,Adela Almasan +groupAttributeSync,2024-09-09T15:29:43Z,,6ded6a8872204a818b3795dc733cc5fe5db066a0,Aaron Godin +kubernetesFolders,2024-09-10T09:22:08Z,,b12a29a1dac8b9aec4a99be08e1665939cb27dc5,Arati R. +alertingFilterV2,2024-09-11T11:29:26Z,,90ee52e8d9c14237f8a57b622c0def7512e657cd,Gilles De Mey +improvedExternalSessionHandling,2024-09-17T10:54:39Z,,41cd0f51800d4849345fc0980ca4173967fc8e9e,Misi +datasourceAPIServers,2024-09-19T08:28:27Z,,f21a5987a22bcdb596d6a258d2960e4151348b63,Ryan McKinley +useSessionStorageForRedirection,2024-09-23T09:31:23Z,,b369341868ec182f076e2e6003b371fdf071f690,Misi +homeSetupGuide,2024-09-25T17:20:04Z,,c822feff9edd6da63da44d935f7cac4365871f01,Serena +alertingQueryAndExpressionsStepMode,2024-09-26T06:33:14Z,,536edee7bff0e393054775b02a9362c8d49ed699,Sonia Aguilar diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index e65a21dd9b5..978bcbdb39b 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -97,6 +97,7 @@ transformationsVariableSupport,GA,@grafana/dataviz-squad,false,false,true kubernetesPlaylists,GA,@grafana/grafana-app-platform-squad,false,true,false kubernetesSnapshots,experimental,@grafana/grafana-app-platform-squad,false,true,false kubernetesDashboards,experimental,@grafana/grafana-app-platform-squad,false,false,true +kubernetesDashboardsAPI,experimental,@grafana/grafana-app-platform-squad,false,true,false kubernetesFolders,experimental,@grafana/search-and-storage,false,false,false grafanaAPIServerTestingWithExperimentalAPIs,experimental,@grafana/search-and-storage,false,false,false datasourceQueryTypes,experimental,@grafana/grafana-app-platform-squad,false,true,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 31fe284f57f..221a413cdb3 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -399,6 +399,10 @@ const ( // Use the kubernetes API in the frontend for dashboards FlagKubernetesDashboards = "kubernetesDashboards" + // FlagKubernetesDashboardsAPI + // Use the kubernetes API in the backend for dashboards + FlagKubernetesDashboardsAPI = "kubernetesDashboardsAPI" + // FlagKubernetesFolders // Use the kubernetes API in the frontend for folders, and route /api/folders requests to k8s FlagKubernetesFolders = "kubernetesFolders" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 40e0388ab28..35140c0d400 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -56,8 +56,8 @@ "metadata": { "name": "adhocFilterOneOf", "resourceVersion": "1723119716623", - "creationTimestamp": "2024-08-08T12:21:56Z", - "deletionTimestamp": "2024-09-05T12:30:12Z" + "creationTimestamp": "2024-08-12T08:56:42Z", + "deletionTimestamp": "2024-09-05T12:49:24Z" }, "spec": { "description": "Exposes a new 'one of' operator for ad-hoc filters. This operator allows users to filter by multiple values in a single filter.", @@ -170,7 +170,7 @@ "metadata": { "name": "alertingFilterV2", "resourceVersion": "1723028774805", - "creationTimestamp": "2024-08-07T11:06:14Z" + "creationTimestamp": "2024-09-11T11:29:26Z" }, "spec": { "description": "Enable the new alerting search experience", @@ -260,7 +260,7 @@ "metadata": { "name": "alertingQueryAndExpressionsStepMode", "resourceVersion": "1725978395461", - "creationTimestamp": "2024-09-10T14:26:35Z" + "creationTimestamp": "2024-09-26T06:33:14Z" }, "spec": { "description": "Enables step mode for alerting queries and expressions", @@ -386,8 +386,8 @@ "metadata": { "name": "appPlatformAccessTokens", "resourceVersion": "1725549369316", - "creationTimestamp": "2024-09-05T15:16:09Z", - "deletionTimestamp": "2024-10-11T15:54:21Z" + "creationTimestamp": "2024-09-05T16:18:44Z", + "deletionTimestamp": "2024-10-14T13:14:46Z" }, "spec": { "description": "Enables the use of access tokens for the App Platform", @@ -415,7 +415,7 @@ "metadata": { "name": "appSidecar", "resourceVersion": "1722872007883", - "creationTimestamp": "2024-08-05T15:33:27Z" + "creationTimestamp": "2024-09-09T12:45:05Z" }, "spec": { "description": "Enable the app sidecar feature that allows rendering 2 apps at the same time", @@ -580,7 +580,7 @@ "name": "awsDatasourcesNewFormStyling", "resourceVersion": "1720021873452", "creationTimestamp": "2023-10-12T08:59:10Z", - "deletionTimestamp": "2024-07-05T10:20:55Z", + "deletionTimestamp": "2024-07-22T12:48:17Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-03 15:51:13.452477 +0000 UTC" } @@ -624,8 +624,8 @@ "metadata": { "name": "backgroundPluginInstaller", "resourceVersion": "1723202510081", - "creationTimestamp": "2024-08-09T11:21:50Z", - "deletionTimestamp": "2024-09-20T13:20:41Z" + "creationTimestamp": "2024-08-12T14:39:31Z", + "deletionTimestamp": "2024-09-23T13:49:18Z" }, "spec": { "description": "Enable background plugin installer", @@ -639,7 +639,7 @@ "name": "bodyScrolling", "resourceVersion": "1721723807004", "creationTimestamp": "2024-07-01T10:28:39Z", - "deletionTimestamp": "2024-09-24T09:17:00Z", + "deletionTimestamp": "2024-09-24T12:23:18Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-23 08:36:47.004393 +0000 UTC" } @@ -927,7 +927,7 @@ "name": "databaseReadReplica", "resourceVersion": "1720021873452", "creationTimestamp": "2024-06-18T15:07:15Z", - "deletionTimestamp": "2024-09-20T20:03:26Z", + "deletionTimestamp": "2024-09-25T23:21:39Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-03 15:51:13.452477 +0000 UTC" } @@ -943,7 +943,7 @@ "metadata": { "name": "dataplaneAggregator", "resourceVersion": "1723151074613", - "creationTimestamp": "2024-08-08T21:04:34Z" + "creationTimestamp": "2024-08-09T08:41:07Z" }, "spec": { "description": "Enable grafana dataplane aggregator", @@ -974,7 +974,7 @@ "metadata": { "name": "datasourceAPIServers", "resourceVersion": "1726731672938", - "creationTimestamp": "2024-09-12T07:32:40Z", + "creationTimestamp": "2024-09-19T08:28:27Z", "annotations": { "grafana.app/updatedTimestamp": "2024-09-19 07:41:12.938146 +0000 UTC" } @@ -1008,7 +1008,7 @@ "name": "datasourceQueryMultiStatus", "resourceVersion": "1718727528075", "creationTimestamp": "2022-05-03T16:02:20Z", - "deletionTimestamp": "2024-07-08T14:46:08Z" + "deletionTimestamp": "2024-07-10T09:15:10Z" }, "spec": { "description": "Introduce HTTP 207 Multi Status for api/ds/query", @@ -1171,7 +1171,7 @@ "metadata": { "name": "exploreLogsAggregatedMetrics", "resourceVersion": "1724938092041", - "creationTimestamp": "2024-08-29T13:28:12Z" + "creationTimestamp": "2024-08-29T13:55:59Z" }, "spec": { "description": "Used in Explore Logs to query by aggregated metrics", @@ -1184,7 +1184,7 @@ "metadata": { "name": "exploreLogsLimitedTimeRange", "resourceVersion": "1724938092041", - "creationTimestamp": "2024-08-29T13:28:12Z" + "creationTimestamp": "2024-08-29T13:55:59Z" }, "spec": { "description": "Used in Explore Logs to limit the time range", @@ -1197,7 +1197,7 @@ "metadata": { "name": "exploreLogsShardSplitting", "resourceVersion": "1724938092041", - "creationTimestamp": "2024-08-29T13:28:12Z" + "creationTimestamp": "2024-08-29T13:55:59Z" }, "spec": { "description": "Used in Explore Logs to split queries into multiple queries based on the number of shards", @@ -1348,7 +1348,7 @@ "name": "flameGraphItemCollapsing", "resourceVersion": "1718727528075", "creationTimestamp": "2023-11-09T14:31:07Z", - "deletionTimestamp": "2024-07-08T14:17:01Z" + "deletionTimestamp": "2024-07-15T12:45:41Z" }, "spec": { "description": "Allow collapsing of flame graph items", @@ -1448,7 +1448,7 @@ "metadata": { "name": "groupAttributeSync", "resourceVersion": "1725893018130", - "creationTimestamp": "2024-09-09T14:43:38Z" + "creationTimestamp": "2024-09-09T15:29:43Z" }, "spec": { "description": "Enable the groupsync extension for managing Group Attribute Sync feature", @@ -1508,7 +1508,7 @@ "metadata": { "name": "homeSetupGuide", "resourceVersion": "1726258153467", - "creationTimestamp": "2024-09-10T15:46:32Z", + "creationTimestamp": "2024-09-25T17:20:04Z", "annotations": { "grafana.app/updatedTimestamp": "2024-09-13 20:09:13.467989 +0000 UTC" } @@ -1525,7 +1525,7 @@ "name": "idForwarding", "resourceVersion": "1718727528075", "creationTimestamp": "2023-09-25T15:21:28Z", - "deletionTimestamp": "2024-08-21T11:35:56Z" + "deletionTimestamp": "2024-08-21T13:30:17Z" }, "spec": { "description": "Generate signed id token for identity that can be forwarded to plugins and external services", @@ -1537,7 +1537,7 @@ "metadata": { "name": "improvedExternalSessionHandling", "resourceVersion": "1726560214520", - "creationTimestamp": "2024-09-17T08:03:34Z" + "creationTimestamp": "2024-09-17T10:54:39Z" }, "spec": { "description": "Enable improved support for external sessions in Grafana", @@ -1644,6 +1644,19 @@ "frontend": true } }, + { + "metadata": { + "name": "kubernetesDashboardsAPI", + "resourceVersion": "1729017284006", + "creationTimestamp": "2024-10-15T18:34:44Z" + }, + "spec": { + "description": "Use the kubernetes API in the backend for dashboards", + "stage": "experimental", + "codeowner": "@grafana/grafana-app-platform-squad", + "requiresRestart": true + } + }, { "metadata": { "name": "kubernetesFeatureToggles", @@ -1662,7 +1675,7 @@ "metadata": { "name": "kubernetesFolders", "resourceVersion": "1725863636605", - "creationTimestamp": "2024-09-09T06:29:38Z", + "creationTimestamp": "2024-09-10T09:22:08Z", "annotations": { "grafana.app/updatedTimestamp": "2024-09-09 06:33:56.605329 +0000 UTC" } @@ -1678,6 +1691,7 @@ "name": "kubernetesPlaylists", "resourceVersion": "1720021873452", "creationTimestamp": "2023-10-05T19:00:36Z", + "deletionTimestamp": "2024-08-13T08:03:28Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-03 15:51:13.452477 +0000 UTC" } @@ -1945,7 +1959,7 @@ "metadata": { "name": "lokiSendDashboardPanelNames", "resourceVersion": "1724089497989", - "creationTimestamp": "2024-08-19T17:44:18Z", + "creationTimestamp": "2024-08-22T19:30:43Z", "annotations": { "grafana.app/updatedTimestamp": "2024-08-19 17:44:57.989565 +0000 UTC" } @@ -2029,7 +2043,7 @@ "metadata": { "name": "mysqlParseTime", "resourceVersion": "1724750152191", - "creationTimestamp": "2024-08-27T09:15:52Z" + "creationTimestamp": "2024-08-27T11:16:04Z" }, "spec": { "description": "Ensure the parseTime flag is set for MySQL driver", @@ -2088,7 +2102,7 @@ "metadata": { "name": "newFiltersUI", "resourceVersion": "1724228641625", - "creationTimestamp": "2024-08-21T08:24:01Z" + "creationTimestamp": "2024-08-30T12:48:13Z" }, "spec": { "description": "Enables new combobox style UI for the Ad hoc filters variable in scenes architecture", @@ -2355,7 +2369,7 @@ "metadata": { "name": "pluginsDetailsRightPanel", "resourceVersion": "1720788722220", - "creationTimestamp": "2024-07-12T08:39:21Z", + "creationTimestamp": "2024-08-13T09:55:30Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-12 12:52:02.22099 +0000 UTC" } @@ -2487,7 +2501,7 @@ "name": "prometheusDataplane", "resourceVersion": "1720021873452", "creationTimestamp": "2023-03-29T15:26:32Z", - "deletionTimestamp": "2024-08-21T13:35:19Z", + "deletionTimestamp": "2024-08-26T12:53:38Z", "annotations": { "grafana.app/updatedTimestamp": "2024-07-03 15:51:13.452477 +0000 UTC" } @@ -2549,7 +2563,7 @@ "metadata": { "name": "prometheusRunQueriesInParallel", "resourceVersion": "1720677541862", - "creationTimestamp": "2024-07-11T05:59:01Z" + "creationTimestamp": "2024-08-12T12:31:39Z" }, "spec": { "description": "Enables running Prometheus queries in parallel", @@ -2814,7 +2828,7 @@ "metadata": { "name": "singleTopNav", "resourceVersion": "1724861961030", - "creationTimestamp": "2024-08-28T16:19:21Z" + "creationTimestamp": "2024-08-29T08:48:32Z" }, "spec": { "description": "Unifies the top search bar and breadcrumb bar into one", @@ -3053,7 +3067,7 @@ "name": "unifiedStorage", "resourceVersion": "1724096690370", "creationTimestamp": "2023-12-06T20:21:21Z", - "deletionTimestamp": "2024-08-21T09:30:06Z", + "deletionTimestamp": "2024-08-21T16:28:30Z", "annotations": { "grafana.app/updatedTimestamp": "2024-08-19 19:44:50.370023815 +0000 UTC" } @@ -3111,7 +3125,7 @@ "metadata": { "name": "useSessionStorageForRedirection", "resourceVersion": "1727082618788", - "creationTimestamp": "2024-09-23T09:10:18Z" + "creationTimestamp": "2024-09-23T09:31:23Z" }, "spec": { "description": "Use session storage for handling the redirection after login", @@ -3123,7 +3137,7 @@ "metadata": { "name": "vizActions", "resourceVersion": "1722461779830", - "creationTimestamp": "2024-07-31T21:36:19Z" + "creationTimestamp": "2024-09-09T14:11:55Z" }, "spec": { "description": "Allow actions in visualizations", diff --git a/pkg/services/ngalert/writer/prom.go b/pkg/services/ngalert/writer/prom.go index ea46b816125..5bfa688c10b 100644 --- a/pkg/services/ngalert/writer/prom.go +++ b/pkg/services/ngalert/writer/prom.go @@ -25,14 +25,18 @@ const backendType = "prometheus" const ( // Fixed error messages MimirDuplicateTimestampError = "err-mimir-sample-duplicate-timestamp" + MimirInvalidLabelError = "err-mimir-label-invalid" // Best effort error messages PrometheusDuplicateTimestampError = "duplicate sample for timestamp" ) var ( - ErrWriteFailure = errors.New("failed to write time series") - ErrBadFrame = errors.New("failed to read dataframe") + // Unexpected, 500-like write errors. + ErrUnexpectedWriteFailure = errors.New("failed to write time series") + // Expected, user-level write errors like trying to write an invalid series. + ErrRejectedWrite = errors.New("series was rejected") + ErrBadFrame = errors.New("failed to read dataframe") ) var DuplicateTimestampErrors = [...]string{ @@ -213,10 +217,12 @@ func (w PrometheusWriter) Write(ctx context.Context, name string, t time.Time, f lvs = append(lvs, fmt.Sprint(res.StatusCode)) w.metrics.WritesTotal.WithLabelValues(lvs...).Inc() - if err, ignored := checkWriteError(writeErr); err != nil { - return errors.Join(ErrWriteFailure, err) - } else if ignored { - l.Debug("Ignored write error", "error", err, "status_code", res.StatusCode) + if writeErr != nil { + if err, ignored := checkWriteError(writeErr); err != nil { + return err + } else if ignored { + l.Debug("Ignored write error", "error", err, "status_code", res.StatusCode) + } } return nil @@ -242,7 +248,12 @@ func checkWriteError(writeErr promremote.WriteError) (err error, ignored bool) { return nil, false } - // special case for 400 status code + // All 500-range statuses are automatically unexpected and not the fault of the data. + if writeErr.StatusCode()/100 == 5 { + return errors.Join(ErrUnexpectedWriteFailure, writeErr), false + } + + // Special case for 400 status code. 400s may be ignorable in the event of HA writers, or the fault of the written data. if writeErr.StatusCode() == 400 { msg := writeErr.Error() // HA may potentially write different values for the same timestamp, so we ignore this error @@ -252,7 +263,16 @@ func checkWriteError(writeErr promremote.WriteError) (err error, ignored bool) { return nil, true } } + + if strings.Contains(msg, MimirInvalidLabelError) { + return errors.Join(ErrRejectedWrite, writeErr), false + } + + // For now, all 400s that are not previously known are considered unexpected. + // TODO: Consider blanket-converting all 400s to be known errors. This should only be done once we are confident this is not a problem with this client. + return errors.Join(ErrUnexpectedWriteFailure, writeErr), false } - return writeErr, false + // All other errors which do not fit into the above categories are also unexpected. + return errors.Join(ErrUnexpectedWriteFailure, writeErr), false } diff --git a/pkg/services/ngalert/writer/prom_test.go b/pkg/services/ngalert/writer/prom_test.go index 3aa49d85831..43297948354 100644 --- a/pkg/services/ngalert/writer/prom_test.go +++ b/pkg/services/ngalert/writer/prom_test.go @@ -166,6 +166,7 @@ func TestPrometheusWriter_Write(t *testing.T) { err := writer.Write(ctx, "test", now, frames, 1, map[string]string{}) require.Error(t, err) require.ErrorIs(t, err, clientErr) + require.ErrorIs(t, err, ErrUnexpectedWriteFailure) }) t.Run("writes expected points", func(t *testing.T) { @@ -204,6 +205,22 @@ func TestPrometheusWriter_Write(t *testing.T) { }) } }) + + t.Run("bad labels fit under the client error category", func(t *testing.T) { + msg := MimirInvalidLabelError + clientErr := testClientWriteError{ + statusCode: http.StatusBadRequest, + msg: &msg, + } + client.writeSeriesFunc = func(ctx context.Context, ts promremote.TSList, opts promremote.WriteOptions) (promremote.WriteResult, promremote.WriteError) { + return promremote.WriteResult{}, clientErr + } + + err := writer.Write(ctx, "test", now, frames, 1, map[string]string{"extra": "label"}) + + require.Error(t, err) + require.ErrorIs(t, err, ErrRejectedWrite) + }) } func extractValue(t *testing.T, frames data.Frames, labels map[string]string, frameType data.FrameType) float64 { diff --git a/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go index ab1eb9f751a..1870f77daa6 100644 --- a/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go +++ b/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go @@ -10,7 +10,6 @@ import ( "github.com/grafana/grafana/pkg/plugins/instrumentationutils" plog "github.com/grafana/grafana/pkg/plugins/log" "github.com/grafana/grafana/pkg/plugins/manager/registry" - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" ) // NewLoggerMiddleware creates a new backend.HandlerMiddleware that will @@ -61,7 +60,7 @@ func (m *LoggerMiddleware) logRequest(ctx context.Context, pCtx backend.PluginCo if err != nil { logParams = append(logParams, "error", err) } - logParams = append(logParams, "statusSource", pluginrequestmeta.StatusSourceFromContext(ctx)) + logParams = append(logParams, "statusSource", backend.ErrorSourceFromContext(ctx)) if status > instrumentationutils.RequestStatusOK { logFunc = ctxLogger.Error @@ -93,7 +92,8 @@ func (m *LoggerMiddleware) QueryData(ctx context.Context, req *backend.QueryData "refID", refID, "status", int(dr.Status), "error", dr.Error, - "statusSource", pluginrequestmeta.StatusSourceFromPluginErrorSource(dr.ErrorSource), + "statusSource", dr.ErrorSource.String(), + "target", m.pluginTarget(ctx, req.PluginContext), } ctxLogger.Error("Partial data response error", logParams...) } diff --git a/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go index 29e7b1ec998..a75f7a43889 100644 --- a/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go +++ b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go @@ -11,7 +11,6 @@ import ( "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/instrumentationutils" "github.com/grafana/grafana/pkg/plugins/manager/registry" - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" ) // pluginMetrics contains the prometheus metrics used by the MetricsMiddleware. @@ -115,7 +114,7 @@ func (m *MetricsMiddleware) instrumentPluginRequest(ctx context.Context, pluginC status, err := fn(ctx) elapsed := time.Since(start) - statusSource := pluginrequestmeta.StatusSourceFromContext(ctx) + statusSource := backend.ErrorSourceFromContext(ctx) endpoint := backend.EndpointFromContext(ctx) pluginRequestDurationWithLabels := m.pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, string(endpoint), target, string(statusSource)) diff --git a/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go index 1af03e7af79..79580bddfaa 100644 --- a/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go +++ b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go @@ -17,7 +17,6 @@ import ( "github.com/grafana/grafana/pkg/plugins/backendplugin" "github.com/grafana/grafana/pkg/plugins/instrumentationutils" "github.com/grafana/grafana/pkg/plugins/manager/fakes" - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" ) const ( @@ -90,7 +89,7 @@ func TestInstrumentationMiddleware(t *testing.T) { require.Equal(t, 1, testutil.CollectAndCount(promRegistry, metricRequestDurationMs)) require.Equal(t, 1, testutil.CollectAndCount(promRegistry, metricRequestDurationS)) - counter := mw.pluginMetrics.pluginRequestCounter.WithLabelValues(pluginID, string(tc.expEndpoint), instrumentationutils.RequestStatusOK.String(), string(backendplugin.TargetUnknown), string(pluginrequestmeta.DefaultStatusSource)) + counter := mw.pluginMetrics.pluginRequestCounter.WithLabelValues(pluginID, string(tc.expEndpoint), instrumentationutils.RequestStatusOK.String(), string(backendplugin.TargetUnknown), string(backend.DefaultErrorSource)) require.Equal(t, 1.0, testutil.ToFloat64(counter)) for _, m := range []string{metricRequestDurationMs, metricRequestDurationS} { require.NoError(t, checkHistogram(promRegistry, m, map[string]string{ @@ -155,12 +154,11 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { })) metricsMw := newMetricsMiddleware(promRegistry, pluginsRegistry) cdt := handlertest.NewHandlerMiddlewareTest(t, handlertest.WithMiddlewares( - NewPluginRequestMetaMiddleware(), backend.HandlerMiddlewareFunc(func(next backend.Handler) backend.Handler { metricsMw.BaseHandler = backend.NewBaseHandler(next) return metricsMw }), - NewStatusSourceMiddleware(), + backend.NewErrorSourceMiddleware(), )) t.Run("Metrics", func(t *testing.T) { @@ -185,12 +183,12 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { for _, tc := range []struct { name string responses map[string]backend.DataResponse - expStatusSource pluginrequestmeta.StatusSource + expStatusSource backend.ErrorSource }{ { "Default status source for ok responses should be plugin", map[string]backend.DataResponse{"A": okResponse}, - pluginrequestmeta.StatusSourcePlugin, + backend.ErrorSourcePlugin, }, { "Plugin errors should have higher priority than downstream errors", @@ -198,12 +196,12 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { "A": pluginErrorResponse, "B": downstreamErrorResponse, }, - pluginrequestmeta.StatusSourcePlugin, + backend.ErrorSourcePlugin, }, { "Errors without ErrorSource should be reported as plugin status source", map[string]backend.DataResponse{"A": legacyErrorResponse}, - pluginrequestmeta.StatusSourcePlugin, + backend.ErrorSourcePlugin, }, { "Downstream errors should have higher priority than ok responses", @@ -211,7 +209,7 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { "A": okResponse, "B": downstreamErrorResponse, }, - pluginrequestmeta.StatusSourceDownstream, + backend.ErrorSourceDownstream, }, { "Plugin errors should have higher priority than ok responses", @@ -219,7 +217,7 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { "A": okResponse, "B": pluginErrorResponse, }, - pluginrequestmeta.StatusSourcePlugin, + backend.ErrorSourcePlugin, }, { "Legacy errors should have higher priority than ok responses", @@ -227,7 +225,7 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { "A": okResponse, "B": legacyErrorResponse, }, - pluginrequestmeta.StatusSourcePlugin, + backend.ErrorSourcePlugin, }, } { t.Run(tc.name, func(t *testing.T) { @@ -242,7 +240,7 @@ func TestInstrumentationMiddlewareStatusSource(t *testing.T) { } _, err := cdt.MiddlewareHandler.QueryData(context.Background(), &backend.QueryDataRequest{PluginContext: pCtx}) require.NoError(t, err) - ctxStatusSource := pluginrequestmeta.StatusSourceFromContext(cdt.QueryDataCtx) + ctxStatusSource := backend.ErrorSourceFromContext(cdt.QueryDataCtx) require.Equal(t, tc.expStatusSource, ctxStatusSource) }) } diff --git a/pkg/services/pluginsintegration/clientmiddleware/plugin_request_meta_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/plugin_request_meta_middleware.go deleted file mode 100644 index d24e5dc1472..00000000000 --- a/pkg/services/pluginsintegration/clientmiddleware/plugin_request_meta_middleware.go +++ /dev/null @@ -1,86 +0,0 @@ -package clientmiddleware - -import ( - "context" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" -) - -// NewPluginRequestMetaMiddleware returns a new backend.HandlerMiddleware that sets up the default -// values for the plugin request meta in the context.Context. All middlewares that are executed -// after this one are be able to access plugin request meta via the pluginrequestmeta package. -func NewPluginRequestMetaMiddleware() backend.HandlerMiddleware { - return backend.HandlerMiddlewareFunc(func(next backend.Handler) backend.Handler { - return &PluginRequestMetaMiddleware{ - BaseHandler: backend.NewBaseHandler(next), - defaultStatusSource: pluginrequestmeta.DefaultStatusSource, - } - }) -} - -type PluginRequestMetaMiddleware struct { - backend.BaseHandler - defaultStatusSource pluginrequestmeta.StatusSource -} - -func (m *PluginRequestMetaMiddleware) withDefaultPluginRequestMeta(ctx context.Context) context.Context { - // Setup plugin request status source - ctx = pluginrequestmeta.WithStatusSource(ctx, m.defaultStatusSource) - - return ctx -} - -func (m *PluginRequestMetaMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.QueryData(ctx, req) -} - -func (m *PluginRequestMetaMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.CallResource(ctx, req, sender) -} - -func (m *PluginRequestMetaMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.CheckHealth(ctx, req) -} - -func (m *PluginRequestMetaMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.CollectMetrics(ctx, req) -} - -func (m *PluginRequestMetaMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.SubscribeStream(ctx, req) -} - -func (m *PluginRequestMetaMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.PublishStream(ctx, req) -} - -func (m *PluginRequestMetaMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.RunStream(ctx, req, sender) -} - -// ValidateAdmission implements backend.AdmissionHandler. -func (m *PluginRequestMetaMiddleware) ValidateAdmission(ctx context.Context, req *backend.AdmissionRequest) (*backend.ValidationResponse, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.ValidateAdmission(ctx, req) -} - -// MutateAdmission implements backend.AdmissionHandler. -func (m *PluginRequestMetaMiddleware) MutateAdmission(ctx context.Context, req *backend.AdmissionRequest) (*backend.MutationResponse, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.MutateAdmission(ctx, req) -} - -// ConvertObject implements backend.AdmissionHandler. -func (m *PluginRequestMetaMiddleware) ConvertObjects(ctx context.Context, req *backend.ConversionRequest) (*backend.ConversionResponse, error) { - ctx = m.withDefaultPluginRequestMeta(ctx) - return m.BaseHandler.ConvertObjects(ctx, req) -} diff --git a/pkg/services/pluginsintegration/clientmiddleware/plugin_request_meta_middleware_test.go b/pkg/services/pluginsintegration/clientmiddleware/plugin_request_meta_middleware_test.go deleted file mode 100644 index b87fcfb84d5..00000000000 --- a/pkg/services/pluginsintegration/clientmiddleware/plugin_request_meta_middleware_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package clientmiddleware - -import ( - "context" - "testing" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana-plugin-sdk-go/backend/handlertest" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" -) - -func TestPluginRequestMetaMiddleware(t *testing.T) { - t.Run("default", func(t *testing.T) { - cdt := handlertest.NewHandlerMiddlewareTest(t, - handlertest.WithMiddlewares(NewPluginRequestMetaMiddleware()), - ) - _, err := cdt.MiddlewareHandler.QueryData(context.Background(), &backend.QueryDataRequest{}) - require.NoError(t, err) - ss := pluginrequestmeta.StatusSourceFromContext(cdt.QueryDataCtx) - require.Equal(t, pluginrequestmeta.StatusSourcePlugin, ss) - }) - - t.Run("other value", func(t *testing.T) { - cdt := handlertest.NewHandlerMiddlewareTest(t, - handlertest.WithMiddlewares(backend.HandlerMiddlewareFunc(func(next backend.Handler) backend.Handler { - return &PluginRequestMetaMiddleware{ - BaseHandler: backend.NewBaseHandler(next), - defaultStatusSource: "test", - } - })), - ) - _, err := cdt.MiddlewareHandler.QueryData(context.Background(), &backend.QueryDataRequest{}) - require.NoError(t, err) - ss := pluginrequestmeta.StatusSourceFromContext(cdt.QueryDataCtx) - require.Equal(t, pluginrequestmeta.StatusSource("test"), ss) - }) -} diff --git a/pkg/services/pluginsintegration/clientmiddleware/status_source_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/status_source_middleware.go deleted file mode 100644 index d9af8df3887..00000000000 --- a/pkg/services/pluginsintegration/clientmiddleware/status_source_middleware.go +++ /dev/null @@ -1,58 +0,0 @@ -package clientmiddleware - -import ( - "context" - "fmt" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" -) - -// NewStatusSourceMiddleware returns a new backend.HandlerMiddleware that sets the status source in the -// plugin request meta stored in the context.Context, according to the query data responses returned by QueryError. -// If at least one query data response has a "downstream" status source and there isn't one with a "plugin" status source, -// the plugin request meta in the context is set to "downstream". -func NewStatusSourceMiddleware() backend.HandlerMiddleware { - return backend.HandlerMiddlewareFunc(func(next backend.Handler) backend.Handler { - return &StatusSourceMiddleware{ - BaseHandler: backend.NewBaseHandler(next), - } - }) -} - -type StatusSourceMiddleware struct { - backend.BaseHandler -} - -func (m *StatusSourceMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { - resp, err := m.BaseHandler.QueryData(ctx, req) - if resp == nil || len(resp.Responses) == 0 { - return resp, err - } - - // Set downstream status source in the context if there's at least one response with downstream status source, - // and if there's no plugin error - var hasPluginError bool - var hasDownstreamError bool - for _, r := range resp.Responses { - if r.Error == nil { - continue - } - if r.ErrorSource == backend.ErrorSourceDownstream { - hasDownstreamError = true - } else { - hasPluginError = true - } - } - - // A plugin error has higher priority than a downstream error, - // so set to downstream only if there's no plugin error - if hasDownstreamError && !hasPluginError { - if err := pluginrequestmeta.WithDownstreamStatusSource(ctx); err != nil { - return resp, fmt.Errorf("failed to set downstream status source: %w", err) - } - } - - return resp, err -} diff --git a/pkg/services/pluginsintegration/clientmiddleware/status_source_middleware_test.go b/pkg/services/pluginsintegration/clientmiddleware/status_source_middleware_test.go deleted file mode 100644 index c3544d4e9df..00000000000 --- a/pkg/services/pluginsintegration/clientmiddleware/status_source_middleware_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package clientmiddleware - -import ( - "context" - "errors" - "testing" - - "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana-plugin-sdk-go/backend/handlertest" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/plugins/pluginrequestmeta" -) - -func TestStatusSourceMiddleware(t *testing.T) { - someErr := errors.New("oops") - - for _, tc := range []struct { - name string - - queryDataResponse *backend.QueryDataResponse - - expStatusSource pluginrequestmeta.StatusSource - }{ - { - name: `no error should be "plugin" status source`, - queryDataResponse: nil, - expStatusSource: pluginrequestmeta.StatusSourcePlugin, - }, - { - name: `single downstream error should be "downstream" status source`, - queryDataResponse: &backend.QueryDataResponse{ - Responses: map[string]backend.DataResponse{ - "A": {Error: someErr, ErrorSource: backend.ErrorSourceDownstream}, - }, - }, - expStatusSource: pluginrequestmeta.StatusSourceDownstream, - }, - { - name: `single plugin error should be "plugin" status source`, - queryDataResponse: &backend.QueryDataResponse{ - Responses: map[string]backend.DataResponse{ - "A": {Error: someErr, ErrorSource: backend.ErrorSourcePlugin}, - }, - }, - expStatusSource: pluginrequestmeta.StatusSourcePlugin, - }, - { - name: `multiple downstream errors should be "downstream" status source`, - queryDataResponse: &backend.QueryDataResponse{ - Responses: map[string]backend.DataResponse{ - "A": {Error: someErr, ErrorSource: backend.ErrorSourceDownstream}, - "B": {Error: someErr, ErrorSource: backend.ErrorSourceDownstream}, - }, - }, - expStatusSource: pluginrequestmeta.StatusSourceDownstream, - }, - { - name: `single plugin error mixed with downstream errors should be "plugin" status source`, - queryDataResponse: &backend.QueryDataResponse{ - Responses: map[string]backend.DataResponse{ - "A": {Error: someErr, ErrorSource: backend.ErrorSourceDownstream}, - "B": {Error: someErr, ErrorSource: backend.ErrorSourcePlugin}, - "C": {Error: someErr, ErrorSource: backend.ErrorSourceDownstream}, - }, - }, - expStatusSource: pluginrequestmeta.StatusSourcePlugin, - }, - } { - t.Run(tc.name, func(t *testing.T) { - cdt := handlertest.NewHandlerMiddlewareTest(t, - handlertest.WithMiddlewares( - NewPluginRequestMetaMiddleware(), - NewStatusSourceMiddleware(), - ), - ) - cdt.TestHandler.QueryDataFunc = func(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { - cdt.QueryDataCtx = ctx - return tc.queryDataResponse, nil - } - - _, _ = cdt.MiddlewareHandler.QueryData(context.Background(), &backend.QueryDataRequest{}) - - ss := pluginrequestmeta.StatusSourceFromContext(cdt.QueryDataCtx) - require.Equal(t, tc.expStatusSource, ss) - }) - } -} diff --git a/pkg/services/pluginsintegration/pluginsintegration.go b/pkg/services/pluginsintegration/pluginsintegration.go index 36c20fa8cbc..7d25ae72446 100644 --- a/pkg/services/pluginsintegration/pluginsintegration.go +++ b/pkg/services/pluginsintegration/pluginsintegration.go @@ -171,7 +171,6 @@ func NewMiddlewareHandler( func CreateMiddlewares(cfg *setting.Cfg, oAuthTokenService oauthtoken.OAuthTokenService, tracer tracing.Tracer, cachingService caching.CachingService, features featuremgmt.FeatureToggles, promRegisterer prometheus.Registerer, registry registry.Service) []backend.HandlerMiddleware { middlewares := []backend.HandlerMiddleware{ - clientmiddleware.NewPluginRequestMetaMiddleware(), clientmiddleware.NewTracingMiddleware(tracer), clientmiddleware.NewMetricsMiddleware(promRegisterer, registry), clientmiddleware.NewContextualLoggerMiddleware(), @@ -202,9 +201,9 @@ func CreateMiddlewares(cfg *setting.Cfg, oAuthTokenService oauthtoken.OAuthToken middlewares = append(middlewares, clientmiddleware.NewHTTPClientMiddleware()) - // StatusSourceMiddleware should be at the very bottom, or any middlewares below it won't see the - // correct status source in their context.Context - middlewares = append(middlewares, clientmiddleware.NewStatusSourceMiddleware()) + // ErrorSourceMiddleware should be at the very bottom, or any middlewares below it won't see the + // correct error source in their context.Context + middlewares = append(middlewares, backend.NewErrorSourceMiddleware()) return middlewares } diff --git a/pkg/services/sqlstore/migrations/user_mig.go b/pkg/services/sqlstore/migrations/user_mig.go index d2c9d319c95..68fe8ba9af4 100644 --- a/pkg/services/sqlstore/migrations/user_mig.go +++ b/pkg/services/sqlstore/migrations/user_mig.go @@ -160,8 +160,7 @@ func addUserMigrations(mg *Migrator) { mg.AddMigration(usermig.AllowSameLoginCrossOrgs, &usermig.ServiceAccountsSameLoginCrossOrgs{}) // Before it was fixed, the previous migration introduced the org_id again in logins that already had it. // This migration removes the duplicate org_id from the login. - // TODO(aarongodin): this migration was causing instances to fail to start. It must be corrected before being added - // mg.AddMigration(usermig.DedupOrgInLogin, &usermig.ServiceAccountsDeduplicateOrgInLogin{}) + mg.AddMigration(usermig.DedupOrgInLogin, &usermig.ServiceAccountsDeduplicateOrgInLogin{}) // Users login and email should be in lower case mg.AddMigration(usermig.LowerCaseUserLoginAndEmail, &usermig.UsersLowerCaseLoginAndEmail{}) diff --git a/pkg/services/sqlstore/migrations/usermig/service_account_multiple_org_login_migrator.go b/pkg/services/sqlstore/migrations/usermig/service_account_multiple_org_login_migrator.go index 54b71250f1b..05436292267 100644 --- a/pkg/services/sqlstore/migrations/usermig/service_account_multiple_org_login_migrator.go +++ b/pkg/services/sqlstore/migrations/usermig/service_account_multiple_org_login_migrator.go @@ -94,27 +94,39 @@ func (p *ServiceAccountsDeduplicateOrgInLogin) Exec(sess *xorm.Session, mg *migr switch dialect.DriverName() { case migrator.Postgres: _, err = sess.Exec(` - UPDATE "user" + UPDATE "user" AS u SET login = 'sa-' || org_id::text || SUBSTRING(login FROM LENGTH('sa-' || org_id::text || '-' || org_id::text)+1) WHERE login IS NOT NULL AND is_service_account = true - AND login LIKE 'sa-' || org_id::text || '-' || org_id::text || '-%'; + AND login LIKE 'sa-' || org_id::text || '-' || org_id::text || '-%' + AND NOT EXISTS ( + SELECT 1 + FROM "user" AS u2 + WHERE u2.login = 'sa-' || u.org_id::text || SUBSTRING(u.login FROM LENGTH('sa-' || u.org_id::text || '-' || u.org_id::text)+1) + );; `) case migrator.MySQL: _, err = sess.Exec(` - UPDATE user - SET login = CONCAT('sa-', org_id, SUBSTRING(login, LENGTH(CONCAT('sa-', org_id, '-', org_id))+1)) - WHERE login IS NOT NULL - AND is_service_account = 1 - AND login LIKE CONCAT('sa-', org_id, '-', org_id, '-%'); + UPDATE user AS u + LEFT JOIN user AS u2 ON u2.login = CONCAT('sa-', u.org_id, SUBSTRING(u.login, LENGTH(CONCAT('sa-', u.org_id, '-', u.org_id))+1)) + SET u.login = CONCAT('sa-', u.org_id, SUBSTRING(u.login, LENGTH(CONCAT('sa-', u.org_id, '-', u.org_id))+1)) + WHERE u.login IS NOT NULL + AND u.is_service_account = 1 + AND u.login LIKE CONCAT('sa-', u.org_id, '-', u.org_id, '-%') + AND u2.login IS NULL; `) case migrator.SQLite: _, err = sess.Exec(` - UPDATE ` + dialect.Quote("user") + ` - SET login = 'sa-' || CAST(org_id AS TEXT) || SUBSTRING(login, LENGTH('sa-'||CAST(org_id AS TEXT)||'-'||CAST(org_id AS TEXT))+1) - WHERE login IS NOT NULL - AND is_service_account = 1 - AND login LIKE 'sa-'||CAST(org_id AS TEXT)||'-'||CAST(org_id AS TEXT)||'-%'; + UPDATE ` + dialect.Quote("user") + ` AS u + SET login = 'sa-' || CAST(u.org_id AS TEXT) || SUBSTRING(u.login, LENGTH('sa-'||CAST(u.org_id AS TEXT)||'-'||CAST(u.org_id AS TEXT))+1) + WHERE u.login IS NOT NULL + AND u.is_service_account = 1 + AND u.login LIKE 'sa-'||CAST(u.org_id AS TEXT)||'-'||CAST(u.org_id AS TEXT)||'-%' + AND NOT EXISTS ( + SELECT 1 + FROM ` + dialect.Quote("user") + `AS u2 + WHERE u2.login = 'sa-' || CAST(u.org_id AS TEXT) || SUBSTRING(u.login, LENGTH('sa-'||CAST(u.org_id AS TEXT)||'-'||CAST(u.org_id AS TEXT))+1) + );; `) default: return fmt.Errorf("dialect not supported: %s", dialect) diff --git a/pkg/services/sqlstore/migrations/usermig/test/service_account_test.go b/pkg/services/sqlstore/migrations/usermig/test/service_account_test.go index 7ef8b475855..7b289e451bc 100644 --- a/pkg/services/sqlstore/migrations/usermig/test/service_account_test.go +++ b/pkg/services/sqlstore/migrations/usermig/test/service_account_test.go @@ -373,6 +373,43 @@ func TestIntegrationServiceAccountDedupOrgMigration(t *testing.T) { }, }, }, + { + desc: "handle conflicts", + serviceAccounts: []*user.User{ + { + ID: 5, + UID: "u5", + Name: "sa-2-conflict", + Login: "sa-2-conflict", + Email: "sa-2-conflict@example.org", + OrgID: 2, + Created: now, + Updated: now, + IsServiceAccount: true, + }, + { + ID: 6, + UID: "u6", + Name: "sa-2b-conflict", + Login: "sa-2-2-conflict", + Email: "sa-2b-conflict@example.org", + OrgID: 2, + Created: now, + Updated: now, + IsServiceAccount: true, + }, + }, + wantServiceAccounts: []*user.User{ + { + ID: 5, + Login: "sa-2-conflict", + }, + { + ID: 6, + Login: "sa-2-2-conflict", + }, + }, + }, } for _, tc := range testCases { diff --git a/pkg/setting/settings_zanzana.go b/pkg/setting/settings_zanzana.go index 158d71d191f..c9179ad9df0 100644 --- a/pkg/setting/settings_zanzana.go +++ b/pkg/setting/settings_zanzana.go @@ -2,6 +2,7 @@ package setting import ( "slices" + "time" ) type ZanzanaMode string @@ -20,11 +21,19 @@ type ZanzanaSettings struct { ListenHTTP bool // OpenFGA http server address which allows to connect with fga cli HttpAddr string - // Number of check requests running concurrently - ConcurrentChecks int64 // If enabled, authorization cheks will be only performed by zanzana. // This bypasses the performance comparison with the legacy system. ZanzanaOnlyEvaluation bool + // Number of concurrent check requests running by Grafana. + ConcurrentChecks int64 + // Enable cache for Check() requests + CheckQueryCache bool + // TTL for cached requests. Default is 10 seconds. + CheckQueryCacheTTL time.Duration + // Max number of results returned by ListObjects() query. Default is 1000. + ListObjectsMaxResults uint32 + // Deadline for the ListObjects() query. Default is 3 seconds. + ListObjectsDeadline time.Duration } func (cfg *Cfg) readZanzanaSettings() { @@ -45,6 +54,10 @@ func (cfg *Cfg) readZanzanaSettings() { s.HttpAddr = sec.Key("http_addr").MustString("127.0.0.1:8080") s.ConcurrentChecks = sec.Key("concurrent_checks").MustInt64(10) s.ZanzanaOnlyEvaluation = sec.Key("zanzana_only_evaluation").MustBool(false) + s.CheckQueryCache = sec.Key("check_query_cache").MustBool(true) + s.CheckQueryCacheTTL = sec.Key("check_query_cache_ttl").MustDuration(10 * time.Second) + s.ListObjectsDeadline = sec.Key("list_objects_deadline").MustDuration(3 * time.Second) + s.ListObjectsMaxResults = uint32(sec.Key("list_objects_max_results").MustUint(1000)) cfg.Zanzana = s } diff --git a/pkg/tests/apis/dashboard/dashboards_test.go b/pkg/tests/apis/dashboard/dashboards_test.go index 5bab84bf236..9031f2d94b9 100644 --- a/pkg/tests/apis/dashboard/dashboards_test.go +++ b/pkg/tests/apis/dashboard/dashboards_test.go @@ -117,10 +117,9 @@ func TestIntegrationDashboardsApp(t *testing.T) { t.Run("with dual writer mode 0", func(t *testing.T) { helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ - AppModeProduction: false, // required for experimental APIs - DisableAnonymous: true, + DisableAnonymous: true, EnableFeatureToggles: []string{ - featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + featuremgmt.FlagKubernetesDashboardsAPI, featuremgmt.FlagKubernetesDashboards, }, UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{ @@ -134,10 +133,9 @@ func TestIntegrationDashboardsApp(t *testing.T) { t.Run("with dual writer mode 1", func(t *testing.T) { helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ - AppModeProduction: false, // required for experimental APIs - DisableAnonymous: true, + DisableAnonymous: true, EnableFeatureToggles: []string{ - featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + featuremgmt.FlagKubernetesDashboardsAPI, featuremgmt.FlagKubernetesDashboards, }, UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{ @@ -151,10 +149,9 @@ func TestIntegrationDashboardsApp(t *testing.T) { t.Run("with dual writer mode 2", func(t *testing.T) { helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ - AppModeProduction: false, // required for experimental APIs - DisableAnonymous: true, + DisableAnonymous: true, EnableFeatureToggles: []string{ - featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + featuremgmt.FlagKubernetesDashboardsAPI, featuremgmt.FlagKubernetesDashboards, }, UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{ @@ -168,10 +165,9 @@ func TestIntegrationDashboardsApp(t *testing.T) { t.Run("with dual writer mode 3", func(t *testing.T) { helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ - AppModeProduction: false, // required for experimental APIs - DisableAnonymous: true, + DisableAnonymous: true, EnableFeatureToggles: []string{ - featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + featuremgmt.FlagKubernetesDashboardsAPI, featuremgmt.FlagKubernetesDashboards, }, UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{ @@ -185,10 +181,9 @@ func TestIntegrationDashboardsApp(t *testing.T) { t.Run("with dual writer mode 4", func(t *testing.T) { helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ - AppModeProduction: false, // required for experimental APIs - DisableAnonymous: true, + DisableAnonymous: true, EnableFeatureToggles: []string{ - featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + featuremgmt.FlagKubernetesDashboardsAPI, featuremgmt.FlagKubernetesDashboards, }, UnifiedStorageConfig: map[string]setting.UnifiedStorageConfig{ @@ -204,7 +199,7 @@ func TestIntegrationDashboardsApp(t *testing.T) { AppModeProduction: false, // required for experimental APIs DisableAnonymous: true, EnableFeatureToggles: []string{ - featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs, // Required to start the example service + featuremgmt.FlagKubernetesDashboardsAPI, // Required to start the example service }, }) diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/query.go b/pkg/tsdb/grafana-testdata-datasource/kinds/query.go index 6b96c1a6ee9..a2fc0bf6f05 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/query.go +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/query.go @@ -38,6 +38,16 @@ const ( ErrorTypeServerPanic ErrorType = "server_panic" ) +// ErrorSource defines model for TestDataQuery.ErrorSource. +// +enum +type ErrorSource string + +// Defines values for ErrorSource. +const ( + ErrorSourcePlugin ErrorSource = "plugin" + ErrorSourceDownstream ErrorSource = "downstream" +) + // TestDataQueryType defines model for TestDataQueryType. // +enum type TestDataQueryType string @@ -50,6 +60,7 @@ const ( TestDataQueryTypeCsvFile TestDataQueryType = "csv_file" TestDataQueryTypeCsvMetricValues TestDataQueryType = "csv_metric_values" TestDataQueryTypeDatapointsOutsideRange TestDataQueryType = "datapoints_outside_range" + TestDataQueryTypeErrorWithSource TestDataQueryType = "error_with_source" TestDataQueryTypeExponentialHeatmapBucketData TestDataQueryType = "exponential_heatmap_bucket_data" TestDataQueryTypeFlameGraph TestDataQueryType = "flame_graph" TestDataQueryTypeGrafanaApi TestDataQueryType = "grafana_api" @@ -92,21 +103,22 @@ type TestDataQuery struct { Channel string `json:"channel,omitempty"` // Drop percentage (the chance we will lose a point 0-100) - DropPercent float64 `json:"dropPercent,omitempty"` - ErrorType ErrorType `json:"errorType,omitempty"` - FlamegraphDiff bool `json:"flamegraphDiff,omitempty"` - LevelColumn bool `json:"levelColumn,omitempty"` - StartValue float64 `json:"startValue,omitempty"` - Spread float64 `json:"spread,omitempty"` - Noise float64 `json:"noise,omitempty"` - Min *float64 `json:"min,omitempty"` - Max *float64 `json:"max,omitempty"` - WithNil bool `json:"withNil,omitempty"` - Lines int64 `json:"lines,omitempty"` - Points [][]any `json:"points,omitempty"` - RawFrameContent string `json:"rawFrameContent,omitempty"` - SeriesCount int `json:"seriesCount,omitempty"` - SpanCount int `json:"spanCount,omitempty"` + DropPercent float64 `json:"dropPercent,omitempty"` + ErrorType ErrorType `json:"errorType,omitempty"` + FlamegraphDiff bool `json:"flamegraphDiff,omitempty"` + LevelColumn bool `json:"levelColumn,omitempty"` + StartValue float64 `json:"startValue,omitempty"` + Spread float64 `json:"spread,omitempty"` + Noise float64 `json:"noise,omitempty"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + WithNil bool `json:"withNil,omitempty"` + Lines int64 `json:"lines,omitempty"` + Points [][]any `json:"points,omitempty"` + RawFrameContent string `json:"rawFrameContent,omitempty"` + SeriesCount int `json:"seriesCount,omitempty"` + SpanCount int `json:"spanCount,omitempty"` + ErrorSource ErrorSource `json:"errorSource,omitempty"` Nodes *NodesQuery `json:"nodes,omitempty"` PulseWave *PulseWaveQuery `json:"pulseWave,omitempty"` diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/query.panel.schema.json b/pkg/tsdb/grafana-testdata-datasource/kinds/query.panel.schema.json index b0a71f0d307..ab679740dd0 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/query.panel.schema.json +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/query.panel.schema.json @@ -73,6 +73,15 @@ "description": "Drop percentage (the chance we will lose a point 0-100)", "type": "number" }, + "errorSource": { + "description": "Possible enum values:\n - `\"plugin\"` \n - `\"downstream\"` ", + "type": "string", + "enum": [ + "plugin", + "downstream" + ], + "x-enum-description": {} + }, "errorType": { "description": "Possible enum values:\n - `\"frontend_exception\"` \n - `\"frontend_observable\"` \n - `\"server_panic\"` ", "type": "string", @@ -220,7 +229,7 @@ "additionalProperties": false }, "scenarioId": { - "description": "Possible enum values:\n - `\"annotations\"` \n - `\"arrow\"` \n - `\"csv_content\"` \n - `\"csv_file\"` \n - `\"csv_metric_values\"` \n - `\"datapoints_outside_range\"` \n - `\"exponential_heatmap_bucket_data\"` \n - `\"flame_graph\"` \n - `\"grafana_api\"` \n - `\"linear_heatmap_bucket_data\"` \n - `\"live\"` \n - `\"logs\"` \n - `\"manual_entry\"` \n - `\"no_data_points\"` \n - `\"node_graph\"` \n - `\"predictable_csv_wave\"` \n - `\"predictable_pulse\"` \n - `\"random_walk\"` \n - `\"random_walk_table\"` \n - `\"random_walk_with_error\"` \n - `\"raw_frame\"` \n - `\"server_error_500\"` \n - `\"simulation\"` \n - `\"slow_query\"` \n - `\"streaming_client\"` \n - `\"table_static\"` \n - `\"trace\"` \n - `\"usa\"` \n - `\"variables-query\"` ", + "description": "Possible enum values:\n - `\"annotations\"` \n - `\"arrow\"` \n - `\"csv_content\"` \n - `\"csv_file\"` \n - `\"csv_metric_values\"` \n - `\"datapoints_outside_range\"` \n - `\"error_with_source\"` \n - `\"exponential_heatmap_bucket_data\"` \n - `\"flame_graph\"` \n - `\"grafana_api\"` \n - `\"linear_heatmap_bucket_data\"` \n - `\"live\"` \n - `\"logs\"` \n - `\"manual_entry\"` \n - `\"no_data_points\"` \n - `\"node_graph\"` \n - `\"predictable_csv_wave\"` \n - `\"predictable_pulse\"` \n - `\"random_walk\"` \n - `\"random_walk_table\"` \n - `\"random_walk_with_error\"` \n - `\"raw_frame\"` \n - `\"server_error_500\"` \n - `\"simulation\"` \n - `\"slow_query\"` \n - `\"streaming_client\"` \n - `\"table_static\"` \n - `\"trace\"` \n - `\"usa\"` \n - `\"variables-query\"` ", "type": "string", "enum": [ "annotations", @@ -229,6 +238,7 @@ "csv_file", "csv_metric_values", "datapoints_outside_range", + "error_with_source", "exponential_heatmap_bucket_data", "flame_graph", "grafana_api", diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/query.request.schema.json b/pkg/tsdb/grafana-testdata-datasource/kinds/query.request.schema.json index c449cc2e4aa..6d3826164e6 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/query.request.schema.json +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/query.request.schema.json @@ -83,6 +83,15 @@ "description": "Drop percentage (the chance we will lose a point 0-100)", "type": "number" }, + "errorSource": { + "description": "Possible enum values:\n - `\"plugin\"` \n - `\"downstream\"` ", + "type": "string", + "enum": [ + "plugin", + "downstream" + ], + "x-enum-description": {} + }, "errorType": { "description": "Possible enum values:\n - `\"frontend_exception\"` \n - `\"frontend_observable\"` \n - `\"server_panic\"` ", "type": "string", @@ -230,7 +239,7 @@ "additionalProperties": false }, "scenarioId": { - "description": "Possible enum values:\n - `\"annotations\"` \n - `\"arrow\"` \n - `\"csv_content\"` \n - `\"csv_file\"` \n - `\"csv_metric_values\"` \n - `\"datapoints_outside_range\"` \n - `\"exponential_heatmap_bucket_data\"` \n - `\"flame_graph\"` \n - `\"grafana_api\"` \n - `\"linear_heatmap_bucket_data\"` \n - `\"live\"` \n - `\"logs\"` \n - `\"manual_entry\"` \n - `\"no_data_points\"` \n - `\"node_graph\"` \n - `\"predictable_csv_wave\"` \n - `\"predictable_pulse\"` \n - `\"random_walk\"` \n - `\"random_walk_table\"` \n - `\"random_walk_with_error\"` \n - `\"raw_frame\"` \n - `\"server_error_500\"` \n - `\"simulation\"` \n - `\"slow_query\"` \n - `\"streaming_client\"` \n - `\"table_static\"` \n - `\"trace\"` \n - `\"usa\"` \n - `\"variables-query\"` ", + "description": "Possible enum values:\n - `\"annotations\"` \n - `\"arrow\"` \n - `\"csv_content\"` \n - `\"csv_file\"` \n - `\"csv_metric_values\"` \n - `\"datapoints_outside_range\"` \n - `\"error_with_source\"` \n - `\"exponential_heatmap_bucket_data\"` \n - `\"flame_graph\"` \n - `\"grafana_api\"` \n - `\"linear_heatmap_bucket_data\"` \n - `\"live\"` \n - `\"logs\"` \n - `\"manual_entry\"` \n - `\"no_data_points\"` \n - `\"node_graph\"` \n - `\"predictable_csv_wave\"` \n - `\"predictable_pulse\"` \n - `\"random_walk\"` \n - `\"random_walk_table\"` \n - `\"random_walk_with_error\"` \n - `\"raw_frame\"` \n - `\"server_error_500\"` \n - `\"simulation\"` \n - `\"slow_query\"` \n - `\"streaming_client\"` \n - `\"table_static\"` \n - `\"trace\"` \n - `\"usa\"` \n - `\"variables-query\"` ", "type": "string", "enum": [ "annotations", @@ -239,6 +248,7 @@ "csv_file", "csv_metric_values", "datapoints_outside_range", + "error_with_source", "exponential_heatmap_bucket_data", "flame_graph", "grafana_api", diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/query.types.json b/pkg/tsdb/grafana-testdata-datasource/kinds/query.types.json index 50b7dca76d1..2a6a17b811f 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/query.types.json +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/query.types.json @@ -8,7 +8,7 @@ { "metadata": { "name": "default", - "resourceVersion": "1711119846950", + "resourceVersion": "1728405292506", "creationTimestamp": "2024-03-01T02:53:35Z" }, "spec": { @@ -56,6 +56,15 @@ "description": "Drop percentage (the chance we will lose a point 0-100)", "type": "number" }, + "errorSource": { + "description": "Possible enum values:\n - `\"plugin\"` \n - `\"downstream\"` ", + "enum": [ + "plugin", + "downstream" + ], + "type": "string", + "x-enum-description": {} + }, "errorType": { "description": "Possible enum values:\n - `\"frontend_exception\"` \n - `\"frontend_observable\"` \n - `\"server_panic\"` ", "enum": [ @@ -142,7 +151,7 @@ "type": "string" }, "scenarioId": { - "description": "Possible enum values:\n - `\"annotations\"` \n - `\"arrow\"` \n - `\"csv_content\"` \n - `\"csv_file\"` \n - `\"csv_metric_values\"` \n - `\"datapoints_outside_range\"` \n - `\"exponential_heatmap_bucket_data\"` \n - `\"flame_graph\"` \n - `\"grafana_api\"` \n - `\"linear_heatmap_bucket_data\"` \n - `\"live\"` \n - `\"logs\"` \n - `\"manual_entry\"` \n - `\"no_data_points\"` \n - `\"node_graph\"` \n - `\"predictable_csv_wave\"` \n - `\"predictable_pulse\"` \n - `\"random_walk\"` \n - `\"random_walk_table\"` \n - `\"random_walk_with_error\"` \n - `\"raw_frame\"` \n - `\"server_error_500\"` \n - `\"simulation\"` \n - `\"slow_query\"` \n - `\"streaming_client\"` \n - `\"table_static\"` \n - `\"trace\"` \n - `\"usa\"` \n - `\"variables-query\"` ", + "description": "Possible enum values:\n - `\"annotations\"` \n - `\"arrow\"` \n - `\"csv_content\"` \n - `\"csv_file\"` \n - `\"csv_metric_values\"` \n - `\"datapoints_outside_range\"` \n - `\"error_with_source\"` \n - `\"exponential_heatmap_bucket_data\"` \n - `\"flame_graph\"` \n - `\"grafana_api\"` \n - `\"linear_heatmap_bucket_data\"` \n - `\"live\"` \n - `\"logs\"` \n - `\"manual_entry\"` \n - `\"no_data_points\"` \n - `\"node_graph\"` \n - `\"predictable_csv_wave\"` \n - `\"predictable_pulse\"` \n - `\"random_walk\"` \n - `\"random_walk_table\"` \n - `\"random_walk_with_error\"` \n - `\"raw_frame\"` \n - `\"server_error_500\"` \n - `\"simulation\"` \n - `\"slow_query\"` \n - `\"streaming_client\"` \n - `\"table_static\"` \n - `\"trace\"` \n - `\"usa\"` \n - `\"variables-query\"` ", "enum": [ "annotations", "arrow", @@ -150,6 +159,7 @@ "csv_file", "csv_metric_values", "datapoints_outside_range", + "error_with_source", "exponential_heatmap_bucket_data", "flame_graph", "grafana_api", diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/query_test.go b/pkg/tsdb/grafana-testdata-datasource/kinds/query_test.go index 584d4876748..522bad3afad 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/query_test.go +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/query_test.go @@ -21,6 +21,7 @@ func TestQueryTypeDefinitions(t *testing.T) { reflect.TypeOf(NodesQueryTypeRandom), // pick an example value (not the root) reflect.TypeOf(StreamingQueryTypeFetch), // pick an example value (not the root) reflect.TypeOf(ErrorTypeServerPanic), // pick an example value (not the root) + reflect.TypeOf(ErrorSourcePlugin), // pick an example value (not the root) reflect.TypeOf(TestDataQueryTypeAnnotations), // pick an example value (not the root) }, }) diff --git a/pkg/tsdb/grafana-testdata-datasource/scenarios.go b/pkg/tsdb/grafana-testdata-datasource/scenarios.go index de265313f54..982da8c8232 100644 --- a/pkg/tsdb/grafana-testdata-datasource/scenarios.go +++ b/pkg/tsdb/grafana-testdata-datasource/scenarios.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "math" "math/rand" @@ -198,6 +199,12 @@ Timestamps will line up evenly on timeStepSeconds (For example, 60 seconds means Name: "Trace", }) + s.registerScenario(&Scenario{ + ID: kinds.TestDataQueryTypeErrorWithSource, + Name: "Error with source", + handler: s.handleErrorWithSourceScenario, + }) + s.queryMux.HandleFunc("", s.handleFallbackScenario) } @@ -663,6 +670,29 @@ func (s *Service) handleLogsScenario(ctx context.Context, req *backend.QueryData return resp, nil } +func (s *Service) handleErrorWithSourceScenario(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + anErr := errors.New("error") + resp := backend.NewQueryDataResponse() + + for _, q := range req.Queries { + model, err := GetJSONModel(q.JSON) + if err != nil { + continue + } + + respD := resp.Responses[q.RefID] + respD.Error = anErr + + if model.ErrorSource == kinds.ErrorSourceDownstream { + respD.Error = backend.DownstreamError(respD.Error) + } + + resp.Responses[q.RefID] = respD + } + + return resp, nil +} + func RandomWalk(query backend.DataQuery, model kinds.TestDataQuery, index int) *data.Frame { rand := rand.New(rand.NewSource(time.Now().UnixNano() + int64(index))) timeWalkerMs := query.TimeRange.From.UnixNano() / int64(time.Millisecond) diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go index 6bafd668590..0464a21121e 100644 --- a/pkg/tsdb/graphite/graphite.go +++ b/pkg/tsdb/graphite/graphite.go @@ -295,6 +295,9 @@ func (s *Service) toDataFrames(logger log.Logger, response *http.Response, origR tags := make(map[string]string) for name, value := range series.Tags { + if name == "name" { + value = target + } switch value := value.(type) { case string: tags[name] = value diff --git a/project.json b/project.json index fc188ab461b..9c2c25d1e8d 100644 --- a/project.json +++ b/project.json @@ -6,8 +6,9 @@ "start": { "dependsOn": [ "themes-generate", + "grafana-icons:generate", { - "projects": ["@grafana-plugins/**"], + "projects": ["tag:scope:plugin"], "target": "build" } ] @@ -15,8 +16,9 @@ "build": { "dependsOn": [ "themes-generate", + "grafana-icons:generate", { - "projects": ["@grafana-plugins/**"], + "projects": ["tag:scope:plugin"], "target": "build" } ], diff --git a/public/api-enterprise-spec.json b/public/api-enterprise-spec.json index 7a4e13be50b..4999e495ab9 100644 --- a/public/api-enterprise-spec.json +++ b/public/api-enterprise-spec.json @@ -4064,7 +4064,7 @@ "type": "string" }, "ErrorSource": { - "$ref": "#/definitions/ErrorSource" + "$ref": "#/definitions/Source" }, "Frames": { "$ref": "#/definitions/Frames" @@ -4361,10 +4361,6 @@ } } }, - "ErrorSource": { - "description": "ErrorSource type defines the source of the error", - "type": "string" - }, "ExplorePanelsState": { "description": "This is an object constructed with the keys as the values of the enum VisType and the value being a bag of properties" }, @@ -7284,6 +7280,10 @@ } } }, + "Source": { + "type": "string", + "title": "Source type defines the status source." + }, "State": { "description": "+enum", "type": "string" @@ -9420,10 +9420,7 @@ "getTeamLBACRulesResponse": { "description": "", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/TeamLBACRules" - } + "$ref": "#/definitions/TeamLBACRules" } }, "getTeamMembersResponse": { diff --git a/public/api-merged.json b/public/api-merged.json index 51e16c1393a..b1bd32864ed 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -14503,7 +14503,7 @@ "type": "string" }, "ErrorSource": { - "$ref": "#/definitions/ErrorSource" + "$ref": "#/definitions/Source" }, "Frames": { "$ref": "#/definitions/Frames" @@ -20478,6 +20478,10 @@ } } }, + "Source": { + "type": "string", + "title": "Source type defines the status source." + }, "Span": { "type": "object", "title": "A Span defines a continuous sequence of buckets.", @@ -23756,10 +23760,7 @@ "getTeamLBACRulesResponse": { "description": "(empty)", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/TeamLBACRules" - } + "$ref": "#/definitions/TeamLBACRules" } }, "getTeamMembersResponse": { diff --git a/public/app/core/components/AppChrome/AppChromeService.tsx b/public/app/core/components/AppChrome/AppChromeService.tsx index c041574aff9..ba7fb666478 100644 --- a/public/app/core/components/AppChrome/AppChromeService.tsx +++ b/public/app/core/components/AppChrome/AppChromeService.tsx @@ -1,5 +1,5 @@ import { useObservable } from 'react-use'; -import { BehaviorSubject } from 'rxjs'; +import { BehaviorSubject, distinctUntilChanged, map } from 'rxjs'; import { AppEvents, NavModel, NavModelItem, PageLayoutType, UrlQueryValue } from '@grafana/data'; import { config, locationService, reportInteraction } from '@grafana/runtime'; @@ -12,6 +12,7 @@ import { KioskMode } from 'app/types'; import { RouteDescriptor } from '../../navigation/types'; import { ReturnToPreviousProps } from './ReturnToPrevious/ReturnToPrevious'; +import { TOP_BAR_LEVEL_HEIGHT } from './types'; export interface AppChromeState { chromeless?: boolean; @@ -56,6 +57,31 @@ export class AppChromeService { returnToPrevious: this.returnToPreviousData, }); + public headerHeightObservable = this.state + .pipe( + map(({ actions, chromeless, kioskMode, searchBarHidden }) => { + if (config.featureToggles.singleTopNav) { + if (kioskMode || chromeless) { + return 0; + } else if (actions) { + return TOP_BAR_LEVEL_HEIGHT * 2; + } else { + return TOP_BAR_LEVEL_HEIGHT; + } + } else { + if (kioskMode || chromeless) { + return 0; + } else if (searchBarHidden) { + return TOP_BAR_LEVEL_HEIGHT; + } else { + return TOP_BAR_LEVEL_HEIGHT * 2; + } + } + }) + ) + // only emit if the state has actually changed + .pipe(distinctUntilChanged()); + public setMatchedRoute(route: RouteDescriptor) { if (this.currentRoute !== route) { this.currentRoute = route; diff --git a/public/app/core/context/GrafanaContext.ts b/public/app/core/context/GrafanaContext.ts index cf3ed6f1f1c..1a82d7f5c3d 100644 --- a/public/app/core/context/GrafanaContext.ts +++ b/public/app/core/context/GrafanaContext.ts @@ -1,10 +1,10 @@ import { createContext, useCallback, useContext } from 'react'; +import { useObservable } from 'react-use'; import { GrafanaConfig } from '@grafana/data'; -import { LocationService, locationService, BackendSrv, config } from '@grafana/runtime'; +import { LocationService, locationService, BackendSrv } from '@grafana/runtime'; import { AppChromeService } from '../components/AppChrome/AppChromeService'; -import { TOP_BAR_LEVEL_HEIGHT } from '../components/AppChrome/types'; import { NewFrontendAssetsChecker } from '../services/NewFrontendAssetsChecker'; import { KeybindingSrv } from '../services/keybindingSrv'; @@ -45,23 +45,5 @@ export function useReturnToPreviousInternal() { export function useChromeHeaderHeight() { const { chrome } = useGrafana(); - const { actions, kioskMode, searchBarHidden, chromeless } = chrome.useState(); - - if (config.featureToggles.singleTopNav) { - if (kioskMode || chromeless) { - return 0; - } else if (actions) { - return TOP_BAR_LEVEL_HEIGHT * 2; - } else { - return TOP_BAR_LEVEL_HEIGHT; - } - } else { - if (kioskMode || chromeless) { - return 0; - } else if (searchBarHidden) { - return TOP_BAR_LEVEL_HEIGHT; - } else { - return TOP_BAR_LEVEL_HEIGHT * 2; - } - } + return useObservable(chrome.headerHeightObservable, 0); } diff --git a/public/app/features/alerting/unified/components/common/TextVariants.tsx b/public/app/features/alerting/unified/components/common/TextVariants.tsx index 3842c2b8287..1df419b883d 100644 --- a/public/app/features/alerting/unified/components/common/TextVariants.tsx +++ b/public/app/features/alerting/unified/components/common/TextVariants.tsx @@ -6,3 +6,4 @@ import { Text } from '@grafana/ui'; export const PrimaryText = ({ content }: { content: string }) => {content}; +export const CodeText = ({ content }: { content: string }) => {content}; diff --git a/public/app/features/alerting/unified/components/contact-points/ContactPoints.test.tsx b/public/app/features/alerting/unified/components/contact-points/ContactPoints.test.tsx index fa70dacd4ab..dea25168694 100644 --- a/public/app/features/alerting/unified/components/contact-points/ContactPoints.test.tsx +++ b/public/app/features/alerting/unified/components/contact-points/ContactPoints.test.tsx @@ -126,6 +126,13 @@ describe('contact points', () => { }); }); + describe('templates tab', () => { + it('shows a warning when a template is misconfigured', async () => { + renderWithProvider(, { initialEntries: ['/?tab=templates'] }); + expect((await screen.findAllByText(/^misconfigured$/i))[0]).toBeInTheDocument(); + }); + }); + it('should show / hide loading states, have all actions enabled', async () => { renderWithProvider(); diff --git a/public/app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.config.mock.json b/public/app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.config.mock.json index 784c8ca555f..0077f89f3a0 100644 --- a/public/app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.config.mock.json +++ b/public/app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.config.mock.json @@ -3,10 +3,13 @@ "slack-template": "{{ define \"slack-template\" }} Custom slack template {{ end }}", "custom-email": "{{ define \"custom-email\" }} Custom email template {{ end }}", "provisioned-template": "{{ define \"provisioned-template\" }} Custom provisioned template {{ end }}", - "template with spaces": "{{ define \"template with spaces\" }} Custom template with spaces in the name {{ end }}" + "template with spaces": "{{ define \"template with spaces\" }} Custom template with spaces in the name {{ end }}", + "misconfigured-template": "{{ define \"misconfigured template\" }} Template that is defined in template_files but not templates {{ end }}", + "misconfigured and provisioned": "{{ define \"misconfigured and provisioned template\" }} Provisioned template that is defined in template_files but not templates {{ end }}" }, "template_file_provenances": { - "provisioned-template": "api" + "provisioned-template": "api", + "misconfigured and provisioned": "api" }, "alertmanager_config": { "route": { diff --git a/public/app/features/alerting/unified/components/contact-points/useNotificationTemplates.ts b/public/app/features/alerting/unified/components/contact-points/useNotificationTemplates.ts index e98aca41847..f1d15f6dfe5 100644 --- a/public/app/features/alerting/unified/components/contact-points/useNotificationTemplates.ts +++ b/public/app/features/alerting/unified/components/contact-points/useNotificationTemplates.ts @@ -26,6 +26,7 @@ export interface NotificationTemplate { title: string; content: string; provenance: string; + missing?: boolean; } const { useGetAlertmanagerConfigurationQuery, useLazyGetAlertmanagerConfigurationQuery } = alertmanagerApi; @@ -84,12 +85,15 @@ function templateGroupToTemplate( } function amConfigToTemplates(config: AlertManagerCortexConfig): NotificationTemplate[] { + const { alertmanager_config } = config; + const { templates = [] } = alertmanager_config; return Object.entries(config.template_files).map(([title, content]) => ({ uid: title, title, content, // Undefined, null or empty string should be converted to PROVENANCE_NONE provenance: (config.template_file_provenances ?? {})[title] || PROVENANCE_NONE, + missing: !templates.includes(title), })); } diff --git a/public/app/features/alerting/unified/components/receivers/NewReceiverView.test.tsx b/public/app/features/alerting/unified/components/receivers/NewReceiverView.test.tsx index a7d1df7ffb5..93a0114bc17 100644 --- a/public/app/features/alerting/unified/components/receivers/NewReceiverView.test.tsx +++ b/public/app/features/alerting/unified/components/receivers/NewReceiverView.test.tsx @@ -97,7 +97,11 @@ describe('alerting API server disabled', () => { ); const testBody = await testRequest?.json(); - const saveBody = await saveRequest?.json(); + const fullSaveBody = await saveRequest?.json(); + + // Only snapshot and check the receivers, as we don't want other tests to break this + // just because we added something new to the mock config + const saveBody = fullSaveBody.alertmanager_config.receivers; expect([testBody]).toMatchSnapshot(); expect([saveBody]).toMatchSnapshot(); diff --git a/public/app/features/alerting/unified/components/receivers/TemplatesTable.tsx b/public/app/features/alerting/unified/components/receivers/TemplatesTable.tsx index 240997db6b7..c644a3b5fb7 100644 --- a/public/app/features/alerting/unified/components/receivers/TemplatesTable.tsx +++ b/public/app/features/alerting/unified/components/receivers/TemplatesTable.tsx @@ -1,8 +1,10 @@ import { Fragment, useState } from 'react'; import { logError } from '@grafana/runtime'; -import { ConfirmModal, useStyles2 } from '@grafana/ui'; +import { Badge, ConfirmModal, Tooltip, useStyles2 } from '@grafana/ui'; import { useAppNotification } from 'app/core/copy/appNotification'; +import { t, Trans } from 'app/core/internationalization'; +import { CodeText } from 'app/features/alerting/unified/components/common/TextVariants'; import { Authorize } from '../../components/Authorize'; import { AlertmanagerAction } from '../../hooks/useAbilities'; @@ -116,8 +118,8 @@ function TemplateRow({ notificationTemplate, idx, alertManagerName, onDeleteClic const [isExpanded, setIsExpanded] = useState(false); const { isProvisioned } = useNotificationTemplateMetadata(notificationTemplate); - const { uid, title: name, content: template } = notificationTemplate; - + const { uid, title: name, content: template, missing } = notificationTemplate; + const misconfiguredBadgeText = t('alerting.templates.misconfigured-badge-text', 'Misconfigured'); return ( @@ -125,7 +127,25 @@ function TemplateRow({ notificationTemplate, idx, alertManagerName, onDeleteClic setIsExpanded(!isExpanded)} /> - {name} {isProvisioned && } + {name} {isProvisioned && }{' '} + {missing && ( + + This template is misconfigured. +
+ + Templates must be defined in both the and{' '} + sections of your alertmanager configuration. + + + } + > + + + +
+ )} {isProvisioned && ( diff --git a/public/app/features/alerting/unified/components/receivers/__snapshots__/NewReceiverView.test.tsx.snap b/public/app/features/alerting/unified/components/receivers/__snapshots__/NewReceiverView.test.tsx.snap index 681e1ee181f..ededb25feb5 100644 --- a/public/app/features/alerting/unified/components/receivers/__snapshots__/NewReceiverView.test.tsx.snap +++ b/public/app/features/alerting/unified/components/receivers/__snapshots__/NewReceiverView.test.tsx.snap @@ -34,142 +34,113 @@ exports[`alerting API server disabled should be able to test and save a receiver exports[`alerting API server disabled should be able to test and save a receiver 2`] = ` [ - { - "alertmanager_config": { - "mute_time_intervals": [], - "receivers": [ + [ + { + "grafana_managed_receiver_configs": [ { - "grafana_managed_receiver_configs": [ - { - "disableResolveMessage": false, - "name": "grafana-default-email", - "secureFields": {}, - "settings": { - "addresses": "gilles.demey@grafana.com", - "singleEmail": false, - }, - "type": "email", - "uid": "xeKQrBrnk", - }, - ], + "disableResolveMessage": false, "name": "grafana-default-email", - }, - { - "grafana_managed_receiver_configs": [ - { - "disableResolveMessage": false, - "name": "provisioned-contact-point", - "provenance": "api", - "secureFields": {}, - "settings": { - "addresses": "gilles.demey@grafana.com", - "singleEmail": false, - }, - "type": "email", - "uid": "s8SdCVjnk", - }, - ], - "name": "provisioned-contact-point", - }, - { - "grafana_managed_receiver_configs": [ - { - "disableResolveMessage": false, - "name": "lotsa-emails", - "secureFields": {}, - "settings": { - "addresses": "gilles.demey+1@grafana.com, gilles.demey+2@grafana.com, gilles.demey+3@grafana.com, gilles.demey+4@grafana.com", - "singleEmail": false, - }, - "type": "email", - "uid": "af306c96-35a2-4d6e-908a-4993e245dbb2", - }, - ], - "name": "lotsa-emails", - }, - { - "grafana_managed_receiver_configs": [ - { - "disableResolveMessage": false, - "name": "Slack with multiple channels", - "secureFields": { - "token": true, - }, - "settings": { - "recipient": "test-alerts", - }, - "type": "slack", - "uid": "c02ad56a-31da-46b9-becb-4348ec0890fd", - }, - { - "disableResolveMessage": false, - "name": "Slack with multiple channels", - "secureFields": { - "token": true, - }, - "settings": { - "recipient": "test-alerts2", - }, - "type": "slack", - "uid": "b286a3be-f690-49e2-8605-b075cbace2df", - }, - ], - "name": "Slack with multiple channels", - }, - { - "grafana_managed_receiver_configs": [ - { - "disableResolveMessage": false, - "name": "Oncall-integration", - "settings": { - "url": "https://oncall-endpoint.example.com", - }, - "type": "oncall", - }, - ], - "name": "OnCall Conctact point", - }, - { - "grafana_managed_receiver_configs": [ - { - "disableResolveMessage": false, - "name": "my new receiver", - "secureSettings": {}, - "settings": { - "addresses": "tester@grafana.com", - "singleEmail": false, - }, - "type": "email", - }, - ], - "name": "my new receiver", - }, - ], - "route": { - "receiver": "grafana-default-email", - "routes": [ - { - "receiver": "provisioned-contact-point", + "secureFields": {}, + "settings": { + "addresses": "gilles.demey@grafana.com", + "singleEmail": false, }, - ], - }, - "templates": [ - "slack-template", - "custom-email", - "provisioned-template", - "template with spaces", + "type": "email", + "uid": "xeKQrBrnk", + }, ], - "time_intervals": [], + "name": "grafana-default-email", }, - "template_file_provenances": { - "provisioned-template": "api", + { + "grafana_managed_receiver_configs": [ + { + "disableResolveMessage": false, + "name": "provisioned-contact-point", + "provenance": "api", + "secureFields": {}, + "settings": { + "addresses": "gilles.demey@grafana.com", + "singleEmail": false, + }, + "type": "email", + "uid": "s8SdCVjnk", + }, + ], + "name": "provisioned-contact-point", }, - "template_files": { - "custom-email": "{{ define "custom-email" }} Custom email template {{ end }}", - "provisioned-template": "{{ define "provisioned-template" }} Custom provisioned template {{ end }}", - "slack-template": "{{ define "slack-template" }} Custom slack template {{ end }}", - "template with spaces": "{{ define "template with spaces" }} Custom template with spaces in the name {{ end }}", + { + "grafana_managed_receiver_configs": [ + { + "disableResolveMessage": false, + "name": "lotsa-emails", + "secureFields": {}, + "settings": { + "addresses": "gilles.demey+1@grafana.com, gilles.demey+2@grafana.com, gilles.demey+3@grafana.com, gilles.demey+4@grafana.com", + "singleEmail": false, + }, + "type": "email", + "uid": "af306c96-35a2-4d6e-908a-4993e245dbb2", + }, + ], + "name": "lotsa-emails", }, - }, + { + "grafana_managed_receiver_configs": [ + { + "disableResolveMessage": false, + "name": "Slack with multiple channels", + "secureFields": { + "token": true, + }, + "settings": { + "recipient": "test-alerts", + }, + "type": "slack", + "uid": "c02ad56a-31da-46b9-becb-4348ec0890fd", + }, + { + "disableResolveMessage": false, + "name": "Slack with multiple channels", + "secureFields": { + "token": true, + }, + "settings": { + "recipient": "test-alerts2", + }, + "type": "slack", + "uid": "b286a3be-f690-49e2-8605-b075cbace2df", + }, + ], + "name": "Slack with multiple channels", + }, + { + "grafana_managed_receiver_configs": [ + { + "disableResolveMessage": false, + "name": "Oncall-integration", + "settings": { + "url": "https://oncall-endpoint.example.com", + }, + "type": "oncall", + }, + ], + "name": "OnCall Conctact point", + }, + { + "grafana_managed_receiver_configs": [ + { + "disableResolveMessage": false, + "name": "my new receiver", + "secureSettings": {}, + "settings": { + "addresses": "tester@grafana.com", + "singleEmail": false, + }, + "type": "email", + }, + ], + "name": "my new receiver", + }, + ], ] `; diff --git a/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.test.tsx b/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.test.tsx index 73b8d455436..03266950cac 100644 --- a/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.test.tsx +++ b/public/app/features/alerting/unified/components/receivers/form/fields/TemplateSelector.test.tsx @@ -2,6 +2,7 @@ import { ReactNode } from 'react'; import { render, screen, userEvent } from 'test/test-utils'; import { CodeEditorProps } from '@grafana/ui/src/components/Monaco/types'; +import alertmanagerConfigMock from 'app/features/alerting/unified/components/contact-points/__mocks__/alertmanager.config.mock.json'; import { setupMswServer } from 'app/features/alerting/unified/mockApi'; import { grantUserPermissions } from 'app/features/alerting/unified/mocks'; import { AlertmanagerProvider } from 'app/features/alerting/unified/state/AlertmanagerContext'; @@ -98,7 +99,7 @@ describe('TemplatesPicker', () => { const input = screen.getByRole('combobox'); expect(screen.queryByText('slack-template')).not.toBeInTheDocument(); await userEvent.click(input); - expect(screen.getAllByRole('option')).toHaveLength(7); // 4 templates in mock plus 3 in the default template + expect(screen.getAllByRole('option')).toHaveLength(Object.keys(alertmanagerConfigMock.template_files).length + 3); // 4 templates in mock plus 3 in the default template const template = screen.getByRole('option', { name: 'slack-template' }); await userEvent.click(template); expect(screen.getByText('slack-template')).toBeInTheDocument(); diff --git a/public/app/features/browse-dashboards/permissions.ts b/public/app/features/browse-dashboards/permissions.ts index d4043434bce..3347fb9f442 100644 --- a/public/app/features/browse-dashboards/permissions.ts +++ b/public/app/features/browse-dashboards/permissions.ts @@ -8,7 +8,7 @@ function checkFolderPermission(action: AccessControlAction, folderDTO?: FolderDT function checkCanCreateFolders(folderDTO?: FolderDTO) { // Can only create a folder if we have permissions and either we're at root or nestedFolders is enabled - if (folderDTO && !config.featureToggles.nestedFolders) { + if (folderDTO && folderDTO.uid !== 'general' && !config.featureToggles.nestedFolders) { return false; } diff --git a/public/app/features/connections/Connections.test.tsx b/public/app/features/connections/Connections.test.tsx index 1d752fda505..42995857cde 100644 --- a/public/app/features/connections/Connections.test.tsx +++ b/public/app/features/connections/Connections.test.tsx @@ -1,7 +1,7 @@ -import { render, RenderResult, screen } from '@testing-library/react'; -import { TestProvider } from 'test/helpers/TestProvider'; +import { RenderResult, screen } from '@testing-library/react'; +import { Route, Routes } from 'react-router-dom-v5-compat'; +import { render } from 'test/test-utils'; -import { locationService } from '@grafana/runtime'; import { contextSrv } from 'app/core/services/context_srv'; import { getMockDataSources } from 'app/features/datasources/__mocks__'; import * as api from 'app/features/datasources/api'; @@ -11,7 +11,7 @@ import { getPluginsStateMock } from '../plugins/admin/__mocks__'; import Connections from './Connections'; import { navIndex } from './__mocks__/store.navIndex.mock'; -import { ROUTE_BASE_ID, ROUTES } from './constants'; +import { ROUTES } from './constants'; jest.mock('app/core/services/context_srv'); jest.mock('app/features/datasources/api'); @@ -21,15 +21,17 @@ jest.mock('@grafana/runtime', () => ({ })); const renderPage = ( - path = `/${ROUTE_BASE_ID}`, + path: string = ROUTES.Base, store = configureStore({ navIndex, plugins: getPluginsStateMock([]) }) ): RenderResult => { - locationService.push(path); - return render( - - - + + } /> + , + { + store, + historyOptions: { initialEntries: [path] }, + } ); }; diff --git a/public/app/features/connections/Connections.tsx b/public/app/features/connections/Connections.tsx index fd0a97b1125..05330a180b5 100644 --- a/public/app/features/connections/Connections.tsx +++ b/public/app/features/connections/Connections.tsx @@ -1,7 +1,5 @@ -import { Route, Switch, useLocation } from 'react-router-dom'; -import { Navigate } from 'react-router-dom-v5-compat'; +import { Navigate, Routes, Route, useLocation } from 'react-router-dom-v5-compat'; -import { DataSourcesRoutesContext } from 'app/features/datasources/state'; import { StoreState, useSelector } from 'app/types'; import { ROUTES } from './constants'; @@ -32,39 +30,43 @@ export default function Connections() { const isAddNewConnectionPageOverridden = Boolean(navIndex['standalone-plugin-page-/connections/add-new-connection']); return ( - - - {/* Redirect to "Add new connection" by default */} - } /> - - - - - + + {/* Redirect to "Add new connection" by default */} + } /> + {/* The route paths need to be relative to the parent path (ROUTES.Base), so we need to remove that part */} + } /> + } /> + } + /> + } /> + } + /> - {/* "Add new connection" page - we don't register a route in case a plugin already registers a standalone page for it */} - {!isAddNewConnectionPageOverridden && ( - - )} - - {/* Redirect from earlier routes to updated routes */} - + {/* "Add new connection" page - we don't register a route in case a plugin already registers a standalone page for it */} + {!isAddNewConnectionPageOverridden && ( } + caseSensitive + path={ROUTES.AddNewConnection.replace(ROUTES.Base, '')} + element={} /> - } /> + )} - {/* Not found */} - } /> - - + {/* Redirect from earlier routes to updated routes */} + } /> + } /> + } + /> + + {/* Not found */} + } /> + ); } diff --git a/public/app/features/connections/hooks/useDataSourceSettingsNav.ts b/public/app/features/connections/hooks/useDataSourceSettingsNav.ts index 1d94698a9a8..cbf26cc5f09 100644 --- a/public/app/features/connections/hooks/useDataSourceSettingsNav.ts +++ b/public/app/features/connections/hooks/useDataSourceSettingsNav.ts @@ -1,4 +1,4 @@ -import { useLocation, useParams } from 'react-router-dom'; +import { useLocation, useParams } from 'react-router-dom-v5-compat'; import { NavModel, NavModelItem } from '@grafana/data'; import { getDataSourceSrv } from '@grafana/runtime'; @@ -9,7 +9,7 @@ import { useGetSingle } from 'app/features/plugins/admin/state/hooks'; import { useSelector } from 'app/types'; export function useDataSourceSettingsNav(pageIdParam?: string) { - const { uid } = useParams<{ uid: string }>(); + const { uid = '' } = useParams<{ uid: string }>(); const location = useLocation(); const datasource = useDataSource(uid); const dataSourceMeta = useDataSourceMeta(datasource.type); diff --git a/public/app/features/connections/pages/DataSourceDashboardsPage.tsx b/public/app/features/connections/pages/DataSourceDashboardsPage.tsx index ecda29195d7..9a82b0f0d15 100644 --- a/public/app/features/connections/pages/DataSourceDashboardsPage.tsx +++ b/public/app/features/connections/pages/DataSourceDashboardsPage.tsx @@ -1,4 +1,4 @@ -import { useParams } from 'react-router-dom'; +import { useParams } from 'react-router-dom-v5-compat'; import { Page } from 'app/core/components/Page/Page'; import { DataSourceDashboards } from 'app/features/datasources/components/DataSourceDashboards'; @@ -6,7 +6,7 @@ import { DataSourceDashboards } from 'app/features/datasources/components/DataSo import { useDataSourceSettingsNav } from '../hooks/useDataSourceSettingsNav'; export function DataSourceDashboardsPage() { - const { uid } = useParams<{ uid: string }>(); + const { uid = '' } = useParams<{ uid: string }>(); const { navId, pageNav } = useDataSourceSettingsNav('dashboards'); return ( diff --git a/public/app/features/connections/pages/DataSourceDetailsPage.tsx b/public/app/features/connections/pages/DataSourceDetailsPage.tsx index 9a72ddb5783..9ed3153764b 100644 --- a/public/app/features/connections/pages/DataSourceDetailsPage.tsx +++ b/public/app/features/connections/pages/DataSourceDetailsPage.tsx @@ -1,4 +1,4 @@ -import { useParams } from 'react-router-dom'; +import { useParams } from 'react-router-dom-v5-compat'; import { Alert, Badge } from '@grafana/ui'; import { PluginDetailsPage } from 'app/features/plugins/admin/components/PluginDetailsPage'; @@ -8,7 +8,7 @@ import { ROUTES } from '../constants'; export function DataSourceDetailsPage() { const overrideNavId = 'standalone-plugin-page-/connections/add-new-connection'; - const { id } = useParams<{ id: string }>(); + const { id = '' } = useParams<{ id: string }>(); const navIndex = useSelector((state: StoreState) => state.navIndex); const isConnectDataPageOverriden = Boolean(navIndex[overrideNavId]); const navId = isConnectDataPageOverriden ? overrideNavId : 'connections-add-new-connection'; // The nav id changes (gets a prefix) if it is overriden by a plugin diff --git a/public/app/features/connections/pages/EditDataSourcePage.tsx b/public/app/features/connections/pages/EditDataSourcePage.tsx index be105116ca0..fe2d93bc888 100644 --- a/public/app/features/connections/pages/EditDataSourcePage.tsx +++ b/public/app/features/connections/pages/EditDataSourcePage.tsx @@ -1,9 +1,9 @@ -import { useLocation, useParams } from 'react-router-dom'; +import { useLocation, useParams } from 'react-router-dom-v5-compat'; import DataSourceTabPage from 'app/features/datasources/components/DataSourceTabPage'; export function EditDataSourcePage() { - const { uid } = useParams<{ uid: string }>(); + const { uid = '' } = useParams<{ uid: string }>(); const location = useLocation(); const params = new URLSearchParams(location.search); const pageId = params.get('page'); diff --git a/public/app/features/connections/routes.tsx b/public/app/features/connections/routes.tsx index d49a322e8f6..935caa40164 100644 --- a/public/app/features/connections/routes.tsx +++ b/public/app/features/connections/routes.tsx @@ -1,12 +1,12 @@ import { SafeDynamicImport } from 'app/core/components/DynamicImports/SafeDynamicImport'; import { RouteDescriptor } from 'app/core/navigation/types'; -import { ROUTE_BASE_ID } from './constants'; +import { ROUTES } from './constants'; export function getRoutes(): RouteDescriptor[] { return [ { - path: `/${ROUTE_BASE_ID}`, + path: ROUTES.Base, exact: false, component: SafeDynamicImport( () => import(/* webpackChunkName: "Connections"*/ 'app/features/connections/Connections') diff --git a/public/app/features/connections/tabs/ConnectData/ConnectData.tsx b/public/app/features/connections/tabs/ConnectData/ConnectData.tsx index 63737f4cdff..bc21502dfcb 100644 --- a/public/app/features/connections/tabs/ConnectData/ConnectData.tsx +++ b/public/app/features/connections/tabs/ConnectData/ConnectData.tsx @@ -1,6 +1,5 @@ import { css } from '@emotion/css'; -import { useMemo, useState } from 'react'; -import * as React from 'react'; +import { useMemo, useState, FormEvent, MouseEvent } from 'react'; import { GrafanaTheme2, PluginType } from '@grafana/data'; import { useStyles2, LoadingPlaceholder, EmptyState } from '@grafana/ui'; @@ -38,7 +37,7 @@ export function AddNewConnection() { const styles = useStyles2(getStyles); const canCreateDataSources = contextSrv.hasPermission(AccessControlAction.DataSourcesCreate); - const handleSearchChange = (e: React.FormEvent) => { + const handleSearchChange = (e: FormEvent) => { setQueryParams({ search: e.currentTarget.value.toLowerCase(), }); @@ -62,7 +61,7 @@ export function AddNewConnection() { [plugins] ); - const onClickCardGridItem = (e: React.MouseEvent, item: CardGridItem) => { + const onClickCardGridItem = (e: MouseEvent, item: CardGridItem) => { if (!canCreateDataSources) { e.preventDefault(); e.stopPropagation(); diff --git a/public/app/features/dashboard-scene/pages/DashboardScenePageStateManager.ts b/public/app/features/dashboard-scene/pages/DashboardScenePageStateManager.ts index 3f7ade11335..b954cdd3ecc 100644 --- a/public/app/features/dashboard-scene/pages/DashboardScenePageStateManager.ts +++ b/public/app/features/dashboard-scene/pages/DashboardScenePageStateManager.ts @@ -255,6 +255,10 @@ export class DashboardScenePageStateManager extends StateManagerBase ({ - href: `/dashboard/${link.title}`, - title: link.title, - tooltip: link.tooltip || null, -})); - -// Mock the getLinkSrv function -jest.mock('app/features/panel/panellinks/link_srv', () => ({ - getLinkSrv: jest.fn(() => ({ - getAnchorInfo: mockGetAnchorInfo, - })), -})); - describe('DashboardControls', () => { describe('Given a standard scene', () => { it('should initialize with default values', () => { - const { controls: scene } = buildTestScene(); + const scene = buildTestScene(); expect(scene.state.variableControls).toEqual([]); expect(scene.state.timePicker).toBeDefined(); expect(scene.state.refreshPicker).toBeDefined(); }); it('should return if time controls are hidden', () => { - const { controls: scene } = buildTestScene({ + const scene = buildTestScene({ hideTimeControls: false, hideVariableControls: false, hideLinksControls: false, @@ -45,14 +31,14 @@ describe('DashboardControls', () => { describe('Component', () => { it('should render', () => { - const { controls: scene } = buildTestScene(); + const scene = buildTestScene(); expect(() => { render(); }).not.toThrow(); }); it('should render visible controls', async () => { - const { controls: scene } = buildTestScene({ + const scene = buildTestScene({ variableControls: [new VariableValueSelectors({}), new SceneDataLayerControls()], }); const renderer = render(); @@ -65,7 +51,7 @@ describe('DashboardControls', () => { }); it('should render with hidden controls', async () => { - const { controls: scene } = buildTestScene({ + const scene = buildTestScene({ hideTimeControls: true, hideVariableControls: true, hideLinksControls: true, @@ -79,13 +65,13 @@ describe('DashboardControls', () => { describe('UrlSync', () => { it('should return keys', () => { - const { controls: scene } = buildTestScene(); + const scene = buildTestScene(); // @ts-expect-error expect(scene._urlSync.getKeys()).toEqual(['_dash.hideTimePicker', '_dash.hideVariables', '_dash.hideLinks']); }); it('should not return url state for hide flags', () => { - const { controls: scene } = buildTestScene(); + const scene = buildTestScene(); expect(scene.getUrlState()).toEqual({}); scene.setState({ hideTimeControls: true, @@ -96,7 +82,7 @@ describe('DashboardControls', () => { }); it('should update from url', () => { - const { controls: scene } = buildTestScene(); + const scene = buildTestScene(); scene.updateFromUrl({ '_dash.hideTimePicker': 'true', '_dash.hideVariables': 'true', @@ -116,7 +102,7 @@ describe('DashboardControls', () => { }); it('should not override state if no new state comes from url', () => { - const { controls: scene } = buildTestScene({ + const scene = buildTestScene({ hideTimeControls: true, hideVariableControls: true, hideLinksControls: true, @@ -128,7 +114,7 @@ describe('DashboardControls', () => { }); it('should not call setState if no changes', () => { - const { controls: scene } = buildTestScene({ + const scene = buildTestScene({ hideTimeControls: true, hideVariableControls: true, hideLinksControls: true, @@ -144,34 +130,9 @@ describe('DashboardControls', () => { expect(setState).toHaveBeenCalledTimes(0); }); }); - - it('Should update link hrefs when time range changes', () => { - const { controls, dashboard } = buildTestScene(); - render(); - - //clear initial calls to getAnchorInfo - mockGetAnchorInfo.mockClear(); - - act(() => { - // Update time range - dashboard.state.$timeRange?.setState({ - value: { - from: toUtc('2021-01-01'), - to: toUtc('2021-01-02'), - raw: { from: toUtc('2020-01-01'), to: toUtc('2020-01-02') }, - }, - }); - }); - - //expect getAnchorInfo to be called after time range change - expect(mockGetAnchorInfo).toHaveBeenCalledTimes(1); - }); }); -function buildTestScene(state?: Partial): { - dashboard: DashboardScene; - controls: DashboardControls; -} { +function buildTestScene(state?: Partial): DashboardControls { const variable = new TextBoxVariable({ name: 'A', label: 'A', @@ -206,5 +167,5 @@ function buildTestScene(state?: Partial): { dashboard.activate(); variable.activate(); - return { dashboard, controls: dashboard.state.controls as DashboardControls }; + return dashboard.state.controls as DashboardControls; } diff --git a/public/app/features/dashboard-scene/scene/DashboardControls.tsx b/public/app/features/dashboard-scene/scene/DashboardControls.tsx index 41e125957a5..78fbc64afd8 100644 --- a/public/app/features/dashboard-scene/scene/DashboardControls.tsx +++ b/public/app/features/dashboard-scene/scene/DashboardControls.tsx @@ -122,10 +122,9 @@ function DashboardControlsRenderer({ model }: SceneComponentProps {!hideVariableControls && variableControls.map((c) => )} - {!hideLinksControls && !editPanel && } + {!hideLinksControls && !editPanel && } {editPanel && } {!hideTimeControls && ( diff --git a/public/app/features/dashboard-scene/scene/DashboardLinksControls.test.tsx b/public/app/features/dashboard-scene/scene/DashboardLinksControls.test.tsx new file mode 100644 index 00000000000..d6d89a2e63f --- /dev/null +++ b/public/app/features/dashboard-scene/scene/DashboardLinksControls.test.tsx @@ -0,0 +1,100 @@ +import { act, render } from '@testing-library/react'; + +import { toUtc } from '@grafana/data'; +import { selectors } from '@grafana/e2e-selectors'; +import { SceneTimeRange } from '@grafana/scenes'; + +import { DashboardControls } from './DashboardControls'; +import { DashboardScene } from './DashboardScene'; + +const mockGetAnchorInfo = jest.fn((link) => ({ + href: `/dashboard/${link.title}`, + title: link.title, + tooltip: link.tooltip || null, +})); + +// Mock the getLinkSrv function +jest.mock('app/features/panel/panellinks/link_srv', () => ({ + getLinkSrv: jest.fn(() => ({ + getAnchorInfo: mockGetAnchorInfo, + })), +})); + +describe('DashboardLinksControls', () => { + it('renders dashboard links correctly', () => { + const { controls } = buildTestScene(); + const renderer = render(); + + // // Expect two dashboard link containers to be rendered + const linkContainers = renderer.getAllByTestId(selectors.components.DashboardLinks.container); + expect(linkContainers).toHaveLength(2); + + // Check link titles and hrefs + const links = renderer.getAllByTestId(selectors.components.DashboardLinks.link); + expect(links[0]).toHaveTextContent('Link 1'); + expect(links[1]).toHaveTextContent('Link 2'); + }); + + it('updates link hrefs when time range changes', () => { + const { controls, dashboard } = buildTestScene(); + render(); + + //clear initial calls to getAnchorInfo + mockGetAnchorInfo.mockClear(); + + act(() => { + // Update time range + dashboard.state.$timeRange?.setState({ + value: { + from: toUtc('2021-01-01'), + to: toUtc('2021-01-02'), + raw: { from: toUtc('2020-01-01'), to: toUtc('2020-01-02') }, + }, + }); + }); + + //expect getAnchorInfo to be called twice, once for each link, after time range change + expect(mockGetAnchorInfo).toHaveBeenCalledTimes(2); + }); +}); + +function buildTestScene(): { controls: DashboardControls; dashboard: DashboardScene } { + const dashboard = new DashboardScene({ + uid: 'A', + links: [ + { + title: 'Link 1', + url: 'http://localhost:3000/$A', + type: 'link', + asDropdown: false, + icon: '', + includeVars: true, + keepTime: true, + tags: [], + targetBlank: false, + tooltip: 'Link 1', + }, + { + title: 'Link 2', + url: 'http://localhost:3000/$A', + type: 'link', + asDropdown: false, + icon: '', + includeVars: true, + keepTime: true, + tags: [], + targetBlank: false, + tooltip: 'Link 2', + }, + ], + controls: new DashboardControls({}), + $timeRange: new SceneTimeRange({ + from: 'now-1', + to: 'now', + }), + }); + + dashboard.activate(); + + return { controls: dashboard.state.controls as DashboardControls, dashboard }; +} diff --git a/public/app/features/dashboard-scene/scene/DashboardLinksControls.tsx b/public/app/features/dashboard-scene/scene/DashboardLinksControls.tsx index 0337580526a..72a046fc6e5 100644 --- a/public/app/features/dashboard-scene/scene/DashboardLinksControls.tsx +++ b/public/app/features/dashboard-scene/scene/DashboardLinksControls.tsx @@ -1,5 +1,6 @@ import { sanitizeUrl } from '@grafana/data/src/text/sanitize'; import { selectors } from '@grafana/e2e-selectors'; +import { sceneGraph } from '@grafana/scenes'; import { DashboardLink } from '@grafana/schema'; import { Tooltip } from '@grafana/ui'; import { @@ -10,12 +11,17 @@ import { getLinkSrv } from 'app/features/panel/panellinks/link_srv'; import { LINK_ICON_MAP } from '../settings/links/utils'; +import { DashboardScene } from './DashboardScene'; + export interface Props { links: DashboardLink[]; - uid?: string; + dashboard: DashboardScene; } -export function DashboardLinksControls({ links, uid }: Props) { +export function DashboardLinksControls({ links, dashboard }: Props) { + sceneGraph.getTimeRange(dashboard).useState(); + const uid = dashboard.state.uid; + if (!links || !uid) { return null; } diff --git a/public/app/features/datasources/components/DataSourceAddButton.tsx b/public/app/features/datasources/components/DataSourceAddButton.tsx index 8d7de1d5cc0..3d5d9fead3f 100644 --- a/public/app/features/datasources/components/DataSourceAddButton.tsx +++ b/public/app/features/datasources/components/DataSourceAddButton.tsx @@ -2,16 +2,14 @@ import { config } from '@grafana/runtime'; import { LinkButton } from '@grafana/ui'; import { contextSrv } from 'app/core/core'; import { Trans } from 'app/core/internationalization'; +import { ROUTES } from 'app/features/connections/constants'; import { AccessControlAction } from 'app/types'; -import { useDataSourcesRoutes } from '../state'; - export function DataSourceAddButton(): JSX.Element | null { const canCreateDataSource = contextSrv.hasPermission(AccessControlAction.DataSourcesCreate); - const dataSourcesRoutes = useDataSourcesRoutes(); return canCreateDataSource ? ( - + Add new data source ) : null; diff --git a/public/app/features/datasources/components/DataSourcesList.tsx b/public/app/features/datasources/components/DataSourcesList.tsx index bf5a3f2f829..e9cf467c97f 100644 --- a/public/app/features/datasources/components/DataSourcesList.tsx +++ b/public/app/features/datasources/components/DataSourcesList.tsx @@ -9,7 +9,8 @@ import { contextSrv } from 'app/core/core'; import { Trans, t } from 'app/core/internationalization'; import { StoreState, AccessControlAction, useSelector } from 'app/types'; -import { getDataSources, getDataSourcesCount, useDataSourcesRoutes, useLoadDataSources } from '../state'; +import { ROUTES } from '../../connections/constants'; +import { getDataSources, getDataSourcesCount, useLoadDataSources } from '../state'; import { trackDataSourcesListViewed } from '../tracking'; import { DataSourcesListCard } from './DataSourcesListCard'; @@ -54,7 +55,6 @@ export function DataSourcesListView({ hasExploreRights, }: ViewProps) { const styles = useStyles2(getStyles); - const dataSourcesRoutes = useDataSourcesRoutes(); const location = useLocation(); useEffect(() => { @@ -69,7 +69,7 @@ export function DataSourcesListView({ + Add data source } diff --git a/public/app/features/datasources/components/DataSourcesListCard.tsx b/public/app/features/datasources/components/DataSourcesListCard.tsx index f93c3d91def..0c77ea501aa 100644 --- a/public/app/features/datasources/components/DataSourcesListCard.tsx +++ b/public/app/features/datasources/components/DataSourcesListCard.tsx @@ -5,7 +5,7 @@ import { DataSourceSettings, GrafanaTheme2 } from '@grafana/data'; import { config } from '@grafana/runtime'; import { Card, LinkButton, Stack, Tag, useStyles2 } from '@grafana/ui'; -import { useDataSourcesRoutes } from '../state'; +import { ROUTES } from '../../connections/constants'; import { trackCreateDashboardClicked, trackExploreClicked } from '../tracking'; import { constructDataSourceExploreUrl } from '../utils'; @@ -16,8 +16,7 @@ export interface Props { } export function DataSourcesListCard({ dataSource, hasWriteRights, hasExploreRights }: Props) { - const dataSourcesRoutes = useDataSourcesRoutes(); - const dsLink = config.appSubUrl + dataSourcesRoutes.Edit.replace(/:uid/gi, dataSource.uid); + const dsLink = config.appSubUrl + ROUTES.DataSourcesEdit.replace(/:uid/gi, dataSource.uid); const styles = useStyles2(getStyles); return ( diff --git a/public/app/features/datasources/components/NewDataSource.tsx b/public/app/features/datasources/components/NewDataSource.tsx index 86501a76477..e24ab0953fa 100644 --- a/public/app/features/datasources/components/NewDataSource.tsx +++ b/public/app/features/datasources/components/NewDataSource.tsx @@ -1,4 +1,4 @@ -import { AnyAction } from 'redux'; +import { Action } from 'redux'; import { DataSourcePluginMeta, PluginType } from '@grafana/data'; import { LinkButton, FilterInput } from '@grafana/ui'; @@ -6,6 +6,7 @@ import PageLoader from 'app/core/components/PageLoader/PageLoader'; import { PluginsErrorsInfo } from 'app/features/plugins/components/PluginsErrorsInfo'; import { DataSourcePluginCategory, StoreState, useDispatch, useSelector } from 'app/types'; +import { ROUTES } from '../../connections/constants'; import { DataSourceCategories } from '../components/DataSourceCategories'; import { DataSourceTypeCardList } from '../components/DataSourceTypeCardList'; import { @@ -13,7 +14,6 @@ import { useLoadDataSourcePlugins, getFilteredDataSourcePlugins, setDataSourceTypeSearchQuery, - useDataSourcesRoutes, } from '../state'; export function NewDataSource() { @@ -45,7 +45,7 @@ export type ViewProps = { searchQuery: string; isLoading: boolean; onAddDataSource: (dataSource: DataSourcePluginMeta) => void; - onSetSearchQuery: (q: string) => AnyAction; + onSetSearchQuery: (q: string) => Action; }; export function NewDataSourceView({ @@ -56,8 +56,6 @@ export function NewDataSourceView({ onAddDataSource, onSetSearchQuery, }: ViewProps) { - const dataSourcesRoutes = useDataSourcesRoutes(); - if (isLoading) { return ; } @@ -68,7 +66,7 @@ export function NewDataSourceView({
- + Cancel
diff --git a/public/app/features/datasources/state/contexts.ts b/public/app/features/datasources/state/contexts.ts deleted file mode 100644 index df94279c2f5..00000000000 --- a/public/app/features/datasources/state/contexts.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { createContext } from 'react'; - -import { DATASOURCES_ROUTES } from '../constants'; -import { DataSourcesRoutes } from '../types'; - -// The purpose of this context is to be able to override the data-sources routes (used for links for example) used under -// the app/features/datasources modules, so we can reuse them more easily in different parts of the application (e.g. under Connections) -export const DataSourcesRoutesContext = createContext(DATASOURCES_ROUTES); diff --git a/public/app/features/datasources/state/hooks.ts b/public/app/features/datasources/state/hooks.ts index 11587a60d42..d2c956a2a4b 100644 --- a/public/app/features/datasources/state/hooks.ts +++ b/public/app/features/datasources/state/hooks.ts @@ -1,4 +1,4 @@ -import { useContext, useEffect } from 'react'; +import { useEffect } from 'react'; import { DataSourcePluginMeta, DataSourceSettings } from '@grafana/data'; import { cleanUpAction } from 'app/core/actions/cleanUp'; @@ -7,6 +7,7 @@ import { contextSrv } from 'app/core/core'; import { AccessControlAction, useDispatch, useSelector } from 'app/types'; import { ShowConfirmModalEvent } from 'app/types/events'; +import { ROUTES } from '../../connections/constants'; import { DataSourceRights } from '../types'; import { constructDataSourceExploreUrl } from '../utils'; @@ -20,7 +21,6 @@ import { updateDataSource, deleteLoadedDataSource, } from './actions'; -import { DataSourcesRoutesContext } from './contexts'; import { initialDataSourceSettingsState } from './reducers'; import { getDataSource, getDataSourceMeta } from './selectors'; @@ -42,9 +42,8 @@ export const useInitDataSourceSettings = (uid: string) => { export const useTestDataSource = (uid: string) => { const dispatch = useDispatch(); - const dataSourcesRoutes = useDataSourcesRoutes(); - return () => dispatch(testDataSource(uid, dataSourcesRoutes.Edit)); + return () => dispatch(testDataSource(uid, ROUTES.DataSourcesEdit)); }; export const useLoadDataSources = () => { @@ -77,10 +76,9 @@ export const useLoadDataSourcePlugins = () => { export const useAddDatasource = () => { const dispatch = useDispatch(); - const dataSourcesRoutes = useDataSourcesRoutes(); return (plugin: DataSourcePluginMeta) => { - dispatch(addDataSource(plugin, dataSourcesRoutes.Edit)); + dispatch(addDataSource(plugin, ROUTES.DataSourcesEdit)); }; }; @@ -136,7 +134,3 @@ export const useDataSourceRights = (uid: string): DataSourceRights => { hasDeleteRights, }; }; - -export const useDataSourcesRoutes = () => { - return useContext(DataSourcesRoutesContext); -}; diff --git a/public/app/features/datasources/state/index.ts b/public/app/features/datasources/state/index.ts index 37a2d6f5cf0..42fae7a84a7 100644 --- a/public/app/features/datasources/state/index.ts +++ b/public/app/features/datasources/state/index.ts @@ -1,6 +1,5 @@ export * from './actions'; export * from './buildCategories'; -export * from './contexts'; export * from './hooks'; export * from './navModel'; export * from './reducers'; diff --git a/public/app/features/migrate-to-cloud/onprem/NameCell.tsx b/public/app/features/migrate-to-cloud/onprem/NameCell.tsx index b978b1d1b27..62a807c3a4c 100644 --- a/public/app/features/migrate-to-cloud/onprem/NameCell.tsx +++ b/public/app/features/migrate-to-cloud/onprem/NameCell.tsx @@ -219,6 +219,12 @@ function ResourceIcon({ resource }: { resource: ResourceTableItem }) { return ; case 'LIBRARY_ELEMENT': return ; + case 'MUTE_TIMING': + return ; + case 'NOTIFICATION_TEMPLATE': + return ; + case 'CONTACT_POINT': + return ; default: return undefined; } diff --git a/public/app/features/migrate-to-cloud/onprem/TypeCell.tsx b/public/app/features/migrate-to-cloud/onprem/TypeCell.tsx index 37e962d7b53..50ada9b4d55 100644 --- a/public/app/features/migrate-to-cloud/onprem/TypeCell.tsx +++ b/public/app/features/migrate-to-cloud/onprem/TypeCell.tsx @@ -13,6 +13,12 @@ export function prettyTypeName(type: ResourceTableItem['type']) { return t('migrate-to-cloud.resource-type.folder', 'Folder'); case 'LIBRARY_ELEMENT': return t('migrate-to-cloud.resource-type.library_element', 'Library Element'); + case 'MUTE_TIMING': + return t('migrate-to-cloud.resource-type.mute_timing', 'Mute Timing'); + case 'NOTIFICATION_TEMPLATE': + return t('migrate-to-cloud.resource-type.notification_template', 'Notification Template'); + case 'CONTACT_POINT': + return t('migrate-to-cloud.resource-type.contact_point', 'Contact Point'); default: return t('migrate-to-cloud.resource-type.unknown', 'Unknown'); } diff --git a/public/app/features/migrate-to-cloud/onprem/useNotifyOnSuccess.tsx b/public/app/features/migrate-to-cloud/onprem/useNotifyOnSuccess.tsx index 705da66b693..aeeec493993 100644 --- a/public/app/features/migrate-to-cloud/onprem/useNotifyOnSuccess.tsx +++ b/public/app/features/migrate-to-cloud/onprem/useNotifyOnSuccess.tsx @@ -52,6 +52,12 @@ function getTranslatedMessage(snapshot: GetSnapshotResponseDto) { types.push(t('migrate-to-cloud.migrated-counts.folders', 'folders')); } else if (type === 'LIBRARY_ELEMENT') { types.push(t('migrate-to-cloud.migrated-counts.library_elements', 'library elements')); + } else if (type === 'MUTE_TIMING') { + types.push(t('migrate-to-cloud.migrated-counts.mute_timings', 'mute timings')); + } else if (type === 'NOTIFICATION_TEMPLATE') { + types.push(t('migrate-to-cloud.migrated-counts.notification_templates', 'notification templates')); + } else if (type === 'CONTACT_POINT') { + types.push(t('migrate-to-cloud.migrated-counts.contact_points', 'contact points')); } distinctItems += 1; diff --git a/public/app/features/plugins/admin/components/GetStartedWithPlugin/GetStartedWithDataSource.tsx b/public/app/features/plugins/admin/components/GetStartedWithPlugin/GetStartedWithDataSource.tsx index 2c45596367f..22ba118489b 100644 --- a/public/app/features/plugins/admin/components/GetStartedWithPlugin/GetStartedWithDataSource.tsx +++ b/public/app/features/plugins/admin/components/GetStartedWithPlugin/GetStartedWithDataSource.tsx @@ -4,7 +4,8 @@ import * as React from 'react'; import { DataSourcePluginMeta } from '@grafana/data'; import { config } from '@grafana/runtime'; import { Button } from '@grafana/ui'; -import { useDataSourcesRoutes, addDataSource } from 'app/features/datasources/state'; +import { ROUTES } from 'app/features/connections/constants'; +import { addDataSource } from 'app/features/datasources/state'; import { useDispatch } from 'app/types'; import { isDataSourceEditor } from '../../permissions'; @@ -16,15 +17,14 @@ type Props = { export function GetStartedWithDataSource({ plugin }: Props): React.ReactElement | null { const dispatch = useDispatch(); - const dataSourcesRoutes = useDataSourcesRoutes(); const onAddDataSource = useCallback(() => { const meta = { name: plugin.name, id: plugin.id, } as DataSourcePluginMeta; - dispatch(addDataSource(meta, dataSourcesRoutes.Edit)); - }, [dispatch, plugin, dataSourcesRoutes]); + dispatch(addDataSource(meta, ROUTES.DataSourcesEdit)); + }, [dispatch, plugin]); if (!isDataSourceEditor()) { return null; diff --git a/public/app/features/plugins/extensions/logs/log.ts b/public/app/features/plugins/extensions/logs/log.ts index a78d42ccea0..fc87ba3019b 100644 --- a/public/app/features/plugins/extensions/logs/log.ts +++ b/public/app/features/plugins/extensions/logs/log.ts @@ -32,10 +32,12 @@ export class ExtensionsLog { } warning(message: string, labels?: Labels): void { + console.warn(message, labels); this.log(LogLevel.warning, message, labels); } error(message: string, labels?: Labels): void { + console.error(message, labels); this.log(LogLevel.error, message, labels); } diff --git a/public/app/features/query/components/QueryGroupOptions.tsx b/public/app/features/query/components/QueryGroupOptions.tsx index 8fc36b47e1e..c949e1767b2 100644 --- a/public/app/features/query/components/QueryGroupOptions.tsx +++ b/public/app/features/query/components/QueryGroupOptions.tsx @@ -1,11 +1,9 @@ -import { css } from '@emotion/css'; -import { PureComponent, ChangeEvent, FocusEvent } from 'react'; -import * as React from 'react'; +import { css, cx } from '@emotion/css'; +import React, { useState, ChangeEvent, FocusEvent, useCallback } from 'react'; -import { rangeUtil, PanelData, DataSourceApi } from '@grafana/data'; -import { Input, InlineFormLabel, stylesFactory, InlineFieldRow, InlineSwitch } from '@grafana/ui'; +import { rangeUtil, PanelData, DataSourceApi, GrafanaTheme2 } from '@grafana/data'; +import { Input, InlineSwitch, useStyles2, InlineLabel } from '@grafana/ui'; import { QueryOperationRow } from 'app/core/components/QueryOperationRow/QueryOperationRow'; -import { config } from 'app/core/config'; import { QueryGroupOptions } from 'app/types'; interface Props { @@ -15,149 +13,142 @@ interface Props { onChange: (options: QueryGroupOptions) => void; } -interface State { - timeRangeFrom: string; - timeRangeShift: string; - timeRangeHide: boolean; - isOpen: boolean; - relativeTimeIsValid: boolean; - timeShiftIsValid: boolean; -} +export const QueryGroupOptionsEditor = React.memo(({ options, dataSource, data, onChange }: Props) => { + const [timeRangeFrom, setTimeRangeFrom] = useState(options.timeRange?.from || ''); + const [timeRangeShift, setTimeRangeShift] = useState(options.timeRange?.shift || ''); + const [timeRangeHide, setTimeRangeHide] = useState(options.timeRange?.hide ?? false); + const [isOpen, setIsOpen] = useState(false); + const [relativeTimeIsValid, setRelativeTimeIsValid] = useState(true); + const [timeShiftIsValid, setTimeShiftIsValid] = useState(true); -export class QueryGroupOptionsEditor extends PureComponent { - constructor(props: Props) { - super(props); + const styles = useStyles2(getStyles); - const { options } = props; + const onRelativeTimeChange = useCallback((event: ChangeEvent) => { + setTimeRangeFrom(event.target.value); + }, []); - this.state = { - timeRangeFrom: options.timeRange?.from || '', - timeRangeShift: options.timeRange?.shift || '', - timeRangeHide: options.timeRange?.hide ?? false, - isOpen: false, - relativeTimeIsValid: true, - timeShiftIsValid: true, - }; - } + const onTimeShiftChange = useCallback((event: ChangeEvent) => { + setTimeRangeShift(event.target.value); + }, []); - onRelativeTimeChange = (event: ChangeEvent) => { - this.setState({ - timeRangeFrom: event.target.value, - }); - }; + const onOverrideTime = useCallback( + (event: FocusEvent) => { + const newValue = emptyToNull(event.target.value); + const isValid = timeRangeValidation(newValue); - onTimeShiftChange = (event: ChangeEvent) => { - this.setState({ - timeRangeShift: event.target.value, - }); - }; + if (isValid && options.timeRange?.from !== newValue) { + onChange({ + ...options, + timeRange: { + ...(options.timeRange ?? {}), + from: newValue, + }, + }); + } - onOverrideTime = (event: FocusEvent) => { - const { options, onChange } = this.props; + setRelativeTimeIsValid(isValid); + }, + [onChange, options] + ); - const newValue = emptyToNull(event.target.value); - const isValid = timeRangeValidation(newValue); + const onTimeShift = useCallback( + (event: FocusEvent) => { + const newValue = emptyToNull(event.target.value); + const isValid = timeRangeValidation(newValue); - if (isValid && options.timeRange?.from !== newValue) { - onChange({ - ...options, - timeRange: { - ...(options.timeRange ?? {}), - from: newValue, - }, - }); - } + if (isValid && options.timeRange?.shift !== newValue) { + onChange({ + ...options, + timeRange: { + ...(options.timeRange ?? {}), + shift: newValue, + }, + }); + } - this.setState({ relativeTimeIsValid: isValid }); - }; + setTimeShiftIsValid(isValid); + }, + [onChange, options] + ); - onTimeShift = (event: FocusEvent) => { - const { options, onChange } = this.props; - - const newValue = emptyToNull(event.target.value); - const isValid = timeRangeValidation(newValue); - - if (isValid && options.timeRange?.shift !== newValue) { - onChange({ - ...options, - timeRange: { - ...(options.timeRange ?? {}), - shift: newValue, - }, - }); - } - - this.setState({ timeShiftIsValid: isValid }); - }; - - onToggleTimeOverride = () => { - const { onChange, options } = this.props; - - this.setState({ timeRangeHide: !this.state.timeRangeHide }, () => { - onChange({ - ...options, - timeRange: { - ...(options.timeRange ?? {}), - hide: this.state.timeRangeHide, - }, - }); - }); - }; - - onCacheTimeoutBlur = (event: ChangeEvent) => { - const { options, onChange } = this.props; + const onToggleTimeOverride = useCallback(() => { + const newTimeRangeHide = !timeRangeHide; + setTimeRangeHide(newTimeRangeHide); onChange({ ...options, - cacheTimeout: emptyToNull(event.target.value), + timeRange: { + ...(options.timeRange ?? {}), + hide: newTimeRangeHide, + }, }); - }; + }, [onChange, options, timeRangeHide]); - onQueryCachingTTLBlur = (event: ChangeEvent) => { - const { options, onChange } = this.props; - - let ttl: number | null = parseInt(event.target.value, 10); - - if (isNaN(ttl) || ttl === 0) { - ttl = null; - } - - onChange({ - ...options, - queryCachingTTL: ttl, - }); - }; - - onMaxDataPointsBlur = (event: ChangeEvent) => { - const { options, onChange } = this.props; - - let maxDataPoints: number | null = parseInt(event.currentTarget.value, 10); - - if (isNaN(maxDataPoints) || maxDataPoints === 0) { - maxDataPoints = null; - } - - if (maxDataPoints !== options.maxDataPoints) { + const onCacheTimeoutBlur = useCallback( + (event: ChangeEvent) => { onChange({ ...options, - maxDataPoints, + cacheTimeout: emptyToNull(event.target.value), }); - } - }; + }, + [onChange, options] + ); + + const onQueryCachingTTLBlur = useCallback( + (event: ChangeEvent) => { + let ttl: number | null = parseInt(event.target.value, 10); + + if (isNaN(ttl) || ttl === 0) { + ttl = null; + } - onMinIntervalBlur = (event: ChangeEvent) => { - const { options, onChange } = this.props; - const minInterval = emptyToNull(event.target.value); - if (minInterval !== options.minInterval) { onChange({ ...options, - minInterval, + queryCachingTTL: ttl, }); - } - }; + }, + [onChange, options] + ); - renderCacheTimeoutOption() { - const { dataSource, options } = this.props; + const onMaxDataPointsBlur = useCallback( + (event: ChangeEvent) => { + let maxDataPoints: number | null = parseInt(event.currentTarget.value, 10); + if (isNaN(maxDataPoints) || maxDataPoints === 0) { + maxDataPoints = null; + } + + if (maxDataPoints !== options.maxDataPoints) { + onChange({ + ...options, + maxDataPoints, + }); + } + }, + [onChange, options] + ); + + const onMinIntervalBlur = useCallback( + (event: ChangeEvent) => { + const minInterval = emptyToNull(event.target.value); + if (minInterval !== options.minInterval) { + onChange({ + ...options, + minInterval, + }); + } + }, + [onChange, options] + ); + + const onOpenOptions = useCallback(() => { + setIsOpen(true); + }, []); + + const onCloseOptions = useCallback(() => { + setIsOpen(false); + }, []); + + const renderCacheTimeoutOption = () => { const tooltip = `If your time series store has a query cache this option can override the default cache timeout. Specify a numeric value in seconds.`; @@ -166,27 +157,23 @@ export class QueryGroupOptionsEditor extends PureComponent { } return ( -
-
- - Cache timeout - - -
-
+ <> + + Cache timeout + + + ); - } - - renderQueryCachingTTLOption() { - const { dataSource, options } = this.props; + }; + const renderQueryCachingTTLOption = () => { const tooltip = `Cache time-to-live: How long results from this queries in this panel will be cached, in milliseconds. Defaults to the TTL in the caching configuration for this datasource.`; if (!dataSource.cachingConfig?.enabled) { @@ -194,129 +181,101 @@ export class QueryGroupOptionsEditor extends PureComponent { } return ( -
-
- - Cache TTL - - -
-
+ <> + Cache TTL + + ); - } + }; - renderMaxDataPointsOption() { - const { data, options } = this.props; + const renderMaxDataPointsOption = () => { const realMd = data.request?.maxDataPoints; const value = options.maxDataPoints ?? ''; const isAuto = value === ''; return ( -
-
- - The maximum data points per series. Used directly by some data sources and used in calculation of auto - interval. With streaming data this value is used for the rolling buffer. - - } - > - Max data points - - - {isAuto && ( + <> + -
=
-
Width of panel
+ The maximum data points per series. Used directly by some data sources and used in calculation of auto + interval. With streaming data this value is used for the rolling buffer. - )} -
-
+ } + > + Max data points + + + {isAuto && ( + <> + = + Width of panel + + )} + ); - } + }; - renderIntervalOption() { - const { data, dataSource, options } = this.props; + const renderIntervalOption = () => { const realInterval = data.request?.interval; const minIntervalOnDs = dataSource.interval ?? 'No limit'; return ( <> -
-
- - A lower limit for the interval. Recommended to be set to write frequency, for example 1m{' '} - if your data is written every minute. Default value can be set in data source settings for most data - sources. - - } - > - Min interval - - -
-
-
-
- - The evaluated interval that is sent to data source and is used in $__interval and{' '} - $__interval_ms. This value is not exactly equal to{' '} - Time range / max data points, it will approximate a series of magic number. - - } - > - Interval - - {realInterval} -
=
-
Time range / max data points
-
-
+ + A lower limit for the interval. Recommended to be set to write frequency, for example 1m if + your data is written every minute. Default value can be set in data source settings for most data sources. + + } + htmlFor="min-interval-input" + > + Min interval + + + + The evaluated interval that is sent to data source and is used in $__interval and{' '} + $__interval_ms. This value is not exactly equal to Time range / max data points, + it will approximate a series of magic number. + + } + > + Interval + + {realInterval} + = + Time range / max data points ); - } - - onOpenOptions = () => { - this.setState({ isOpen: true }); }; - onCloseOptions = () => { - this.setState({ isOpen: false }); - }; - - renderCollapsedText(styles: StylesType): React.ReactNode | undefined { - const { data, options } = this.props; - const { isOpen } = this.state; - + const renderCollapsedText = (): React.ReactNode | undefined => { if (isOpen) { return undefined; } @@ -326,120 +285,135 @@ export class QueryGroupOptionsEditor extends PureComponent { mdDesc = `auto = ${data.request.maxDataPoints}`; } - let intervalDesc = options.minInterval; - if (data.request) { - intervalDesc = `${data.request.interval}`; - } + const intervalDesc = data.request?.interval ?? options.minInterval; return ( <> - {
MD = {mdDesc}
} - {
Interval = {intervalDesc}
} + {MD = {mdDesc}} + {Interval = {intervalDesc}} ); - } - - render() { - const { timeRangeHide: hideTimeOverride, relativeTimeIsValid, timeShiftIsValid } = this.state; - const { timeRangeFrom: relativeTime, timeRangeShift: timeShift, isOpen } = this.state; - const styles = getStyles(); - - return ( - - {this.renderMaxDataPointsOption()} - {this.renderIntervalOption()} - {this.renderCacheTimeoutOption()} - {this.renderQueryCachingTTLOption()} - -
- - Overrides the relative time range for individual panels, which causes them to be different than what is - selected in the dashboard time picker in the top-right corner of the dashboard. For example to configure - the Last 5 minutes the Relative time should be now-5m and 5m, or variables - like $_relativeTime. - - } - > - Relative time - - -
- -
- - Overrides the time range for individual panels by shifting its start and end relative to the time - picker. For example to configure the Last 1h the Time shift should be now-1h and{' '} - 1h, or variables like $_timeShift. - - } - > - Time shift - - -
- {(timeShift || relativeTime) && ( - - Hide time info - - - )} -
- ); - } -} - -const timeRangeValidation = (value: string | null) => { - if (!value) { - return true; - } - - return rangeUtil.isValidTimeSpan(value); -}; - -const emptyToNull = (value: string) => { - return value === '' ? null : value; -}; - -const getStyles = stylesFactory(() => { - const { theme } = config; - - return { - collapsedText: css` - margin-left: ${theme.spacing.md}; - font-size: ${theme.typography.size.sm}; - color: ${theme.colors.textWeak}; - `, }; + + return ( + +
+ {renderMaxDataPointsOption()} + {renderIntervalOption()} + {renderCacheTimeoutOption()} + {renderQueryCachingTTLOption()} + + + Overrides the relative time range for individual panels, which causes them to be different than what is + selected in the dashboard time picker in the top-right corner of the dashboard. For example to configure + the Last 5 minutes the Relative time should be now-5m and 5m, or variables like{' '} + $_relativeTime. + + } + > + Relative time + + + + Overrides the time range for individual panels by shifting its start and end relative to the time picker. + For example to configure the Last 1h the Time shift should be now-1h and 1h, or + variables like $_timeShift. + + } + > + Time shift + + + {(timeRangeShift || timeRangeFrom) && ( + <> + + Hide time info + + + + )} +
+
+ ); }); -type StylesType = ReturnType; +QueryGroupOptionsEditor.displayName = 'QueryGroupOptionsEditor'; + +function timeRangeValidation(value: string | null) { + return !value || rangeUtil.isValidTimeSpan(value); +} + +function emptyToNull(value: string) { + return value === '' ? null : value; +} + +function getStyles(theme: GrafanaTheme2) { + return { + grid: css({ + display: 'grid', + gridTemplateColumns: `auto minmax(5em, 1fr) auto 1fr`, + gap: theme.spacing(0.5), + gridAutoRows: theme.spacing(4), + whiteSpace: 'nowrap', + }), + firstColumn: css({ + gridColumn: 1, + }), + collapsedText: css({ + marginLeft: theme.spacing(2), + fontSize: theme.typography.size.sm, + color: theme.colors.text.secondary, + }), + noSquish: css({ + display: 'flex', + alignItems: 'center', + padding: theme.spacing(0, 1), + fontWeight: theme.typography.fontWeightMedium, + fontSize: theme.typography.size.sm, + backgroundColor: theme.colors.background.secondary, + borderRadius: theme.shape.radius.default, + }), + left: css({ + justifySelf: 'left', + }), + operator: css({ + color: theme.v1.palette.orange, + }), + }; +} diff --git a/public/app/plugins/datasource/azuremonitor/project.json b/public/app/plugins/datasource/azuremonitor/project.json new file mode 100644 index 00000000000..4247352791d --- /dev/null +++ b/public/app/plugins/datasource/azuremonitor/project.json @@ -0,0 +1,9 @@ +{ + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:plugin", "type:datasource"], + "targets": { + "build": {}, + "dev": {} + } +} diff --git a/public/app/plugins/datasource/cloud-monitoring/project.json b/public/app/plugins/datasource/cloud-monitoring/project.json new file mode 100644 index 00000000000..4247352791d --- /dev/null +++ b/public/app/plugins/datasource/cloud-monitoring/project.json @@ -0,0 +1,9 @@ +{ + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:plugin", "type:datasource"], + "targets": { + "build": {}, + "dev": {} + } +} diff --git a/public/app/plugins/datasource/grafana-postgresql-datasource/project.json b/public/app/plugins/datasource/grafana-postgresql-datasource/project.json new file mode 100644 index 00000000000..4247352791d --- /dev/null +++ b/public/app/plugins/datasource/grafana-postgresql-datasource/project.json @@ -0,0 +1,9 @@ +{ + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:plugin", "type:datasource"], + "targets": { + "build": {}, + "dev": {} + } +} diff --git a/public/app/plugins/datasource/grafana-pyroscope-datasource/project.json b/public/app/plugins/datasource/grafana-pyroscope-datasource/project.json new file mode 100644 index 00000000000..4247352791d --- /dev/null +++ b/public/app/plugins/datasource/grafana-pyroscope-datasource/project.json @@ -0,0 +1,9 @@ +{ + "$schema": "../../../../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "tags": ["scope:plugin", "type:datasource"], + "targets": { + "build": {}, + "dev": {} + } +} diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/QueryEditor.tsx b/public/app/plugins/datasource/grafana-testdata-datasource/QueryEditor.tsx index 8c4e6599073..66eb259dbde 100644 --- a/public/app/plugins/datasource/grafana-testdata-datasource/QueryEditor.tsx +++ b/public/app/plugins/datasource/grafana-testdata-datasource/QueryEditor.tsx @@ -10,6 +10,7 @@ import { CSVContentEditor } from './components/CSVContentEditor'; import { CSVFileEditor } from './components/CSVFileEditor'; import { CSVWavesEditor } from './components/CSVWaveEditor'; import ErrorEditor from './components/ErrorEditor'; +import ErrorWithSourceQueryEditor from './components/ErrorWithSourceEditor'; import { GrafanaLiveEditor } from './components/GrafanaLiveEditor'; import { NodeGraphEditor } from './components/NodeGraphEditor'; import { PredictablePulseEditor } from './components/PredictablePulseEditor'; @@ -120,6 +121,9 @@ export const QueryEditor = ({ query, datasource, onChange, onRunQuery }: Props) update.usa = { mode: usaQueryModes[0].value, }; + break; + case TestDataQueryType.ErrorWithSource: + update.errorSource = 'plugin'; } onUpdate(update); @@ -379,6 +383,9 @@ export const QueryEditor = ({ query, datasource, onChange, onRunQuery }: Props) /> )} + {scenarioId === TestDataQueryType.ErrorWithSource && ( + + )} {description &&

{description}

} diff --git a/public/app/plugins/datasource/grafana-testdata-datasource/components/ErrorWithSourceEditor.tsx b/public/app/plugins/datasource/grafana-testdata-datasource/components/ErrorWithSourceEditor.tsx new file mode 100644 index 00000000000..a013a68b6bf --- /dev/null +++ b/public/app/plugins/datasource/grafana-testdata-datasource/components/ErrorWithSourceEditor.tsx @@ -0,0 +1,32 @@ +import { InlineField, InlineFieldRow, Select } from '@grafana/ui'; + +import { EditorProps } from '../QueryEditor'; + +const OPTIONS = [ + { + label: 'Plugin', + value: 'plugin', + }, + { + label: 'Downstream', + value: 'downstream', + }, +]; + +const ErrorWithSourceQueryEditor = ({ query, onChange }: EditorProps) => { + return ( + + +